@filoz/repair-cli 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +3 -3
- package/dist/src/api.d.ts +35 -0
- package/dist/src/api.d.ts.map +1 -0
- package/dist/src/api.js +118 -0
- package/dist/src/api.js.map +1 -0
- package/dist/src/cli.js +2 -0
- package/dist/src/cli.js.map +1 -1
- package/dist/src/commands/datasets.d.ts +3 -5
- package/dist/src/commands/datasets.d.ts.map +1 -1
- package/dist/src/commands/datasets.js +24 -33
- package/dist/src/commands/datasets.js.map +1 -1
- package/dist/src/commands/providers.d.ts +3 -5
- package/dist/src/commands/providers.d.ts.map +1 -1
- package/dist/src/commands/providers.js +32 -33
- package/dist/src/commands/providers.js.map +1 -1
- package/dist/src/commands/repair.d.ts +3 -5
- package/dist/src/commands/repair.d.ts.map +1 -1
- package/dist/src/commands/repair.js +3 -3
- package/dist/src/commands/repair.js.map +1 -1
- package/dist/src/commands/replicate.d.ts +3 -5
- package/dist/src/commands/replicate.d.ts.map +1 -1
- package/dist/src/commands/replicate.js +3 -3
- package/dist/src/commands/replicate.js.map +1 -1
- package/dist/src/commands/second-copy.d.ts +23 -0
- package/dist/src/commands/second-copy.d.ts.map +1 -0
- package/dist/src/commands/second-copy.js +175 -0
- package/dist/src/commands/second-copy.js.map +1 -0
- package/dist/src/commands/session-key.d.ts +3 -5
- package/dist/src/commands/session-key.d.ts.map +1 -1
- package/dist/src/commands/setup.d.ts.map +1 -1
- package/dist/src/commands/setup.js +11 -32
- package/dist/src/commands/setup.js.map +1 -1
- package/dist/src/commands/wallet.d.ts +3 -5
- package/dist/src/commands/wallet.d.ts.map +1 -1
- package/dist/src/db/dedupe-cids.d.ts +6 -7
- package/dist/src/db/dedupe-cids.d.ts.map +1 -1
- package/dist/src/db/dedupe-cids.js +2 -14
- package/dist/src/db/dedupe-cids.js.map +1 -1
- package/dist/src/db/find-repair-dataset.d.ts +3 -3
- package/dist/src/db/find-repair-dataset.d.ts.map +1 -1
- package/dist/src/db/find-repair-dataset.js +16 -14
- package/dist/src/db/find-repair-dataset.js.map +1 -1
- package/dist/src/db/get-active-data-set-ids.d.ts +9 -0
- package/dist/src/db/get-active-data-set-ids.d.ts.map +1 -0
- package/dist/src/db/get-active-data-set-ids.js +9 -0
- package/dist/src/db/get-active-data-set-ids.js.map +1 -0
- package/dist/src/db/get-alternate-provider-urls.d.ts +6 -0
- package/dist/src/db/get-alternate-provider-urls.d.ts.map +1 -0
- package/dist/src/db/get-alternate-provider-urls.js +24 -0
- package/dist/src/db/get-alternate-provider-urls.js.map +1 -0
- package/dist/src/db/get-pieces.d.ts +8 -27
- package/dist/src/db/get-pieces.d.ts.map +1 -1
- package/dist/src/db/get-pieces.js +44 -97
- package/dist/src/db/get-pieces.js.map +1 -1
- package/dist/src/db/get-repair-provider.d.ts +2 -2
- package/dist/src/db/get-repair-provider.d.ts.map +1 -1
- package/dist/src/db/get-repair-provider.js +8 -20
- package/dist/src/db/get-repair-provider.js.map +1 -1
- package/dist/src/db/repair-create.d.ts.map +1 -1
- package/dist/src/db/repair-create.js +5 -3
- package/dist/src/db/repair-create.js.map +1 -1
- package/dist/src/db/replicate-create.d.ts.map +1 -1
- package/dist/src/db/replicate-create.js +15 -21
- package/dist/src/db/replicate-create.js.map +1 -1
- package/dist/src/db/second-copy-create.d.ts +10 -0
- package/dist/src/db/second-copy-create.d.ts.map +1 -0
- package/dist/src/db/second-copy-create.js +104 -0
- package/dist/src/db/second-copy-create.js.map +1 -0
- package/dist/src/local-schema.d.ts +41 -1
- package/dist/src/local-schema.d.ts.map +1 -1
- package/dist/src/local-schema.js +2 -0
- package/dist/src/local-schema.js.map +1 -1
- package/dist/src/middleware.d.ts +7 -11
- package/dist/src/middleware.d.ts.map +1 -1
- package/dist/src/middleware.js +6 -10
- package/dist/src/middleware.js.map +1 -1
- package/dist/src/pipeline/add-pieces.d.ts +3 -3
- package/dist/src/pipeline/add-pieces.d.ts.map +1 -1
- package/dist/src/pipeline/add-pieces.js +15 -7
- package/dist/src/pipeline/add-pieces.js.map +1 -1
- package/dist/src/pipeline/create-datasets.d.ts +4 -4
- package/dist/src/pipeline/create-datasets.d.ts.map +1 -1
- package/dist/src/pipeline/create-datasets.js +7 -14
- package/dist/src/pipeline/create-datasets.js.map +1 -1
- package/dist/src/types.d.ts +6 -7
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/utils.d.ts +7 -10
- package/dist/src/utils.d.ts.map +1 -1
- package/dist/src/utils.js +21 -2
- package/dist/src/utils.js.map +1 -1
- package/package.json +3 -3
- package/readme.md +98 -13
- package/src/api.ts +220 -0
- package/src/cli.ts +2 -0
- package/src/commands/datasets.ts +24 -36
- package/src/commands/providers.ts +34 -42
- package/src/commands/repair.ts +3 -3
- package/src/commands/replicate.ts +3 -3
- package/src/commands/second-copy.ts +187 -0
- package/src/commands/setup.ts +12 -35
- package/src/db/dedupe-cids.ts +7 -24
- package/src/db/find-repair-dataset.ts +17 -28
- package/src/db/get-active-data-set-ids.ts +29 -0
- package/src/db/get-alternate-provider-urls.ts +43 -0
- package/src/db/get-pieces.ts +82 -214
- package/src/db/get-repair-provider.ts +10 -30
- package/src/db/repair-create.ts +5 -3
- package/src/db/replicate-create.ts +15 -21
- package/src/db/second-copy-create.ts +147 -0
- package/src/local-schema.ts +3 -0
- package/src/middleware.ts +7 -11
- package/src/pipeline/add-pieces.ts +21 -11
- package/src/pipeline/create-datasets.ts +9 -29
- package/src/types.ts +9 -7
- package/src/utils.ts +22 -4
- package/dist/src/db/find-providers-by-cid.d.ts +0 -9
- package/dist/src/db/find-providers-by-cid.d.ts.map +0 -1
- package/dist/src/db/find-providers-by-cid.js +0 -43
- package/dist/src/db/find-providers-by-cid.js.map +0 -1
- package/dist/src/indexer-schema.d.ts +0 -549
- package/dist/src/indexer-schema.d.ts.map +0 -1
- package/dist/src/indexer-schema.js +0 -56
- package/dist/src/indexer-schema.js.map +0 -1
- package/src/db/find-providers-by-cid.ts +0 -70
- package/src/indexer-schema.ts +0 -77
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises'
|
|
2
|
+
import { taskLog } from '@clack/prompts'
|
|
3
|
+
import { getBlockNumber } from 'viem/actions'
|
|
4
|
+
import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
|
|
5
|
+
import type { OperationInsert } from '../local-schema.ts'
|
|
6
|
+
import type { Context } from '../types.ts'
|
|
7
|
+
import { getRepairProvider } from './get-repair-provider.ts'
|
|
8
|
+
|
|
9
|
+
const OPERATIONS_INSERT_BATCH_SIZE = 500
|
|
10
|
+
|
|
11
|
+
export interface SecondCopyCreateOptions extends Context {
|
|
12
|
+
piecesFile: string
|
|
13
|
+
sourceProviderUrl: string
|
|
14
|
+
targetProviderId: bigint
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
type PiecesFile = {
|
|
18
|
+
aggregates: Array<{ piece_cid?: unknown }>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Normalize a provider service URL for storage and equality checks.
|
|
23
|
+
*/
|
|
24
|
+
export function normalizeProviderUrl(url: string): string {
|
|
25
|
+
let parsed: URL
|
|
26
|
+
try {
|
|
27
|
+
parsed = new URL(url)
|
|
28
|
+
} catch {
|
|
29
|
+
throw new RepairCreationError(`Invalid source provider URL: ${url}`)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
33
|
+
throw new RepairCreationError(`Source provider URL must be http(s): ${url}`)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Drop trailing slash so endpoint comparisons are stable.
|
|
37
|
+
return parsed.toString().replace(/\/$/, '')
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Parse a second-copy pieces file and return unique aggregate piece CIDs.
|
|
42
|
+
*/
|
|
43
|
+
export async function parsePieceCidsFromFile(piecesFile: string): Promise<string[]> {
|
|
44
|
+
let raw: unknown
|
|
45
|
+
try {
|
|
46
|
+
raw = JSON.parse(await readFile(piecesFile, 'utf8'))
|
|
47
|
+
} catch (error) {
|
|
48
|
+
throw new RepairCreationError(
|
|
49
|
+
error instanceof Error ? `Failed to read pieces file: ${error.message}` : 'Failed to read pieces file'
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (raw == null || typeof raw !== 'object' || !Array.isArray((raw as PiecesFile).aggregates)) {
|
|
54
|
+
throw new RepairCreationError('Pieces file must contain an "aggregates" array')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const cids = new Set<string>()
|
|
58
|
+
for (const aggregate of (raw as PiecesFile).aggregates) {
|
|
59
|
+
if (typeof aggregate?.piece_cid !== 'string' || aggregate.piece_cid.length === 0) {
|
|
60
|
+
throw new RepairCreationError('Each aggregate must include a non-empty "piece_cid" string')
|
|
61
|
+
}
|
|
62
|
+
cids.add(aggregate.piece_cid)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (cids.size === 0) {
|
|
66
|
+
throw new RepairCreationError('Pieces file contains no piece CIDs')
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return [...cids]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Prepare a second-copy job from a pieces file and an explicit source provider URL.
|
|
74
|
+
*
|
|
75
|
+
* @param options - The options for creating a second-copy job.
|
|
76
|
+
* @returns The ID of the created local job.
|
|
77
|
+
*/
|
|
78
|
+
export async function secondCopyCreate(options: SecondCopyCreateOptions): Promise<number> {
|
|
79
|
+
const { api, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
|
|
80
|
+
const localSchema = localDb._.fullSchema
|
|
81
|
+
const now = Date.now()
|
|
82
|
+
const blockNumber = await getBlockNumber(client)
|
|
83
|
+
const normalizedSourceUrl = normalizeProviderUrl(sourceProviderUrl)
|
|
84
|
+
|
|
85
|
+
const log = taskLog({
|
|
86
|
+
title: 'Creating second-copy',
|
|
87
|
+
limit: 10,
|
|
88
|
+
retainLog: true,
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
const targetProvider = await getRepairProvider({
|
|
92
|
+
api,
|
|
93
|
+
providerId: targetProviderId,
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
if (!targetProvider) {
|
|
97
|
+
throw new NoAlternateProviderError(targetProviderId)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const normalizedTargetUrl = normalizeProviderUrl(targetProvider.serviceUrl)
|
|
101
|
+
if (normalizedSourceUrl === normalizedTargetUrl) {
|
|
102
|
+
throw new RepairCreationError('Source provider URL must differ from the target provider URL')
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const pieceCids = await parsePieceCidsFromFile(piecesFile)
|
|
106
|
+
log.message(`Loaded ${pieceCids.length} unique piece CIDs from ${piecesFile}`)
|
|
107
|
+
|
|
108
|
+
const [repair] = await localDb
|
|
109
|
+
.insert(localSchema.repairs)
|
|
110
|
+
.values({
|
|
111
|
+
kind: 'second_copy',
|
|
112
|
+
repairProviderId: 0n,
|
|
113
|
+
repairDataSetId: null,
|
|
114
|
+
sourceProviderUrl: normalizedSourceUrl,
|
|
115
|
+
targetProviderId: targetProvider.providerId,
|
|
116
|
+
targetProviderUrl: targetProvider.serviceUrl,
|
|
117
|
+
targetDataSetId: null,
|
|
118
|
+
blockNumber,
|
|
119
|
+
createdAt: now,
|
|
120
|
+
updatedAt: now,
|
|
121
|
+
})
|
|
122
|
+
.returning({ id: localSchema.repairs.id })
|
|
123
|
+
|
|
124
|
+
if (!repair) throw new RepairCreationError('Failed to create second-copy row')
|
|
125
|
+
|
|
126
|
+
let totalOperations = 0
|
|
127
|
+
for (let offset = 0; offset < pieceCids.length; offset += OPERATIONS_INSERT_BATCH_SIZE) {
|
|
128
|
+
const batchCids = pieceCids.slice(offset, offset + OPERATIONS_INSERT_BATCH_SIZE)
|
|
129
|
+
const operations: OperationInsert[] = batchCids.map((cid) => ({
|
|
130
|
+
repairId: repair.id,
|
|
131
|
+
type: 'add_piece',
|
|
132
|
+
status: 'pending',
|
|
133
|
+
cid,
|
|
134
|
+
metadata: {},
|
|
135
|
+
alternateProvider: normalizedSourceUrl,
|
|
136
|
+
createdAt: now,
|
|
137
|
+
updatedAt: now,
|
|
138
|
+
}))
|
|
139
|
+
|
|
140
|
+
await localDb.insert(localSchema.operations).values(operations)
|
|
141
|
+
totalOperations += operations.length
|
|
142
|
+
log.message(`Inserted ${operations.length} operations (${totalOperations}/${pieceCids.length})`)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
log.success(`Created second-copy ${repair.id} with ${totalOperations} operations`, { showLog: true })
|
|
146
|
+
return repair.id
|
|
147
|
+
}
|
package/src/local-schema.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { customType, sqliteTable as table } from 'drizzle-orm/sqlite-core'
|
|
|
6
6
|
import * as Json from 'iso-base/json'
|
|
7
7
|
|
|
8
8
|
export type RepairStatus = 'pending' | 'completed' | 'failed'
|
|
9
|
+
export type RepairKind = 'repair' | 'replicate' | 'second_copy'
|
|
9
10
|
export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped'
|
|
10
11
|
export type OperationType = 'create_dataset' | 'add_piece'
|
|
11
12
|
|
|
@@ -44,8 +45,10 @@ export type RepairUpdate = Partial<RepairInsert>
|
|
|
44
45
|
export const repairs = table('repairs', {
|
|
45
46
|
id: t.int().primaryKey({ autoIncrement: true }),
|
|
46
47
|
status: t.text().$type<RepairStatus>().notNull().default('pending'),
|
|
48
|
+
kind: t.text().$type<RepairKind>().notNull().default('repair'),
|
|
47
49
|
repairProviderId: bigintType('repair_provider_id').notNull(),
|
|
48
50
|
repairDataSetId: bigintType('repair_data_set_id'),
|
|
51
|
+
sourceProviderUrl: t.text('source_provider_url'),
|
|
49
52
|
targetProviderId: bigintType('target_provider_id').notNull(),
|
|
50
53
|
targetProviderUrl: t.text('target_provider_url').notNull(),
|
|
51
54
|
targetDataSetId: bigintType('target_data_set_id'),
|
package/src/middleware.ts
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { Chain } from '@filoz/synapse-core/chains'
|
|
2
|
-
import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres'
|
|
3
2
|
import { middleware, z } from 'incur'
|
|
4
3
|
import type { Account, Client, Transport } from 'viem'
|
|
5
|
-
import
|
|
6
|
-
import type {
|
|
4
|
+
import { createIndexerApi } from './api.ts'
|
|
5
|
+
import type { IndexerApi, LocalDatabase } from './types.ts'
|
|
7
6
|
import { config, createLocalDatabase, getClient } from './utils.ts'
|
|
8
7
|
|
|
9
8
|
export const contextSchema = z.object({
|
|
10
|
-
|
|
9
|
+
api: z.custom<IndexerApi>(),
|
|
11
10
|
localDb: z.custom<LocalDatabase>(),
|
|
12
11
|
config: z.custom<typeof config>(),
|
|
13
12
|
client: z.custom<Client<Transport, Chain, Account>>(),
|
|
@@ -17,9 +16,9 @@ export const contextSchema = z.object({
|
|
|
17
16
|
})
|
|
18
17
|
|
|
19
18
|
export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
|
|
20
|
-
const { dbPath, chainId,
|
|
19
|
+
const { dbPath, chainId, apiUrl, source } = config.store
|
|
21
20
|
|
|
22
|
-
if (!dbPath || !chainId || !
|
|
21
|
+
if (!dbPath || !chainId || !source) {
|
|
23
22
|
return c.error({
|
|
24
23
|
code: 'CONFIG_NOT_SET',
|
|
25
24
|
message: 'Config not set. Please run `repair setup` first.',
|
|
@@ -28,13 +27,11 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
|
|
|
28
27
|
}
|
|
29
28
|
|
|
30
29
|
const localDb = await createLocalDatabase(dbPath)
|
|
31
|
-
const
|
|
32
|
-
schema: indexerSchema,
|
|
33
|
-
})
|
|
30
|
+
const api = createIndexerApi({ baseUrl: apiUrl, chainId })
|
|
34
31
|
|
|
35
32
|
const { client, chain } = getClient(chainId)
|
|
36
33
|
c.set('localDb', localDb)
|
|
37
|
-
c.set('
|
|
34
|
+
c.set('api', api)
|
|
38
35
|
c.set('config', config)
|
|
39
36
|
c.set('client', client)
|
|
40
37
|
c.set('chain', chain)
|
|
@@ -43,5 +40,4 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
|
|
|
43
40
|
await next()
|
|
44
41
|
|
|
45
42
|
localDb.$client.close()
|
|
46
|
-
await indexerDb.$client.end()
|
|
47
43
|
})
|
|
@@ -3,17 +3,18 @@ import * as Piece from '@filoz/synapse-core/piece'
|
|
|
3
3
|
import * as SP from '@filoz/synapse-core/sp'
|
|
4
4
|
import { and, asc, eq, gt, inArray } from 'drizzle-orm'
|
|
5
5
|
import PQueue from 'p-queue'
|
|
6
|
+
import { listDataSetPieces } from '../api.ts'
|
|
6
7
|
import { dedupeCids } from '../db/dedupe-cids.ts'
|
|
7
8
|
import { getTargetDataset } from '../db/get-target-dataset.ts'
|
|
8
9
|
import { repairUpdate } from '../db/repair-update.ts'
|
|
9
10
|
import { upsertOperations } from '../db/upsert-operations.ts'
|
|
10
11
|
import type { OperationSelect, RepairSelect } from '../local-schema.ts'
|
|
11
|
-
import type {
|
|
12
|
+
import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
|
|
12
13
|
import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
|
|
13
14
|
|
|
14
15
|
export type RunPullPiecesPhaseOptions = {
|
|
15
16
|
localDb: LocalDatabase
|
|
16
|
-
|
|
17
|
+
api: IndexerApi
|
|
17
18
|
repair: RepairSelect
|
|
18
19
|
concurrency: number
|
|
19
20
|
batchSize: number
|
|
@@ -22,7 +23,8 @@ export type RunPullPiecesPhaseOptions = {
|
|
|
22
23
|
|
|
23
24
|
type CreateAddPiecesWorkerOptions = {
|
|
24
25
|
localDb: LocalDatabase
|
|
25
|
-
|
|
26
|
+
/** CIDs already in the target dataset; repairs only. Updated as batches are added. */
|
|
27
|
+
targetCids?: Set<string>
|
|
26
28
|
repair: RepairSelect
|
|
27
29
|
client: WalletClient
|
|
28
30
|
state: {
|
|
@@ -42,12 +44,11 @@ type AddPiecesJobOptions = {
|
|
|
42
44
|
/**
|
|
43
45
|
* Create a worker function that adds pieces to the target repair dataset.
|
|
44
46
|
*/
|
|
45
|
-
function createAddPiecesWorker({ localDb,
|
|
47
|
+
function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
|
|
46
48
|
return async (options: AddPiecesJobOptions) => {
|
|
47
49
|
let completedOps = 0
|
|
48
50
|
let failedOps = 0
|
|
49
51
|
let operations: OperationSelect[] = options.operations
|
|
50
|
-
const isRepair = repair.repairDataSetId == null
|
|
51
52
|
|
|
52
53
|
const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
|
|
53
54
|
|
|
@@ -55,8 +56,8 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
|
|
|
55
56
|
const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
|
|
56
57
|
|
|
57
58
|
// dedupe operations by CID on the target dataset for repairs jobs
|
|
58
|
-
if (
|
|
59
|
-
operations = await dedupeCids({
|
|
59
|
+
if (targetCids) {
|
|
60
|
+
operations = await dedupeCids({ localDb, existingCids: targetCids, operations })
|
|
60
61
|
}
|
|
61
62
|
// A previous run may have submitted a transaction and crashed before marking rows completed.
|
|
62
63
|
const operationsBeforeConfirmationCheck = operations.length
|
|
@@ -75,7 +76,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
|
|
|
75
76
|
dataSetId: dataset.dataSetId,
|
|
76
77
|
clientDataSetId: dataset.clientDataSetId,
|
|
77
78
|
pieces: operationsToPullPieces(operations),
|
|
78
|
-
timeout: 1000 * 60 * 30,
|
|
79
|
+
timeout: 1000 * 60 * 30, // 30 minutes
|
|
79
80
|
onStatus: (status) => {
|
|
80
81
|
const completed = status.pieces.filter((piece) => piece.status === 'complete').length
|
|
81
82
|
const failed = status.pieces.filter((piece) => piece.status === 'failed').length
|
|
@@ -111,7 +112,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
|
|
|
111
112
|
clientDataSetId: dataset.clientDataSetId,
|
|
112
113
|
pieces: operations.map((operation) => ({
|
|
113
114
|
pieceCid: Piece.from(operation.cid),
|
|
114
|
-
metadata:
|
|
115
|
+
metadata: targetCids ? undefined : operation.metadata,
|
|
115
116
|
})),
|
|
116
117
|
})
|
|
117
118
|
await upsertOperations({
|
|
@@ -126,6 +127,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
|
|
|
126
127
|
const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
|
|
127
128
|
state.completedOperations += operations.length
|
|
128
129
|
completedOps += operations.length
|
|
130
|
+
for (const operation of operations) targetCids?.add(operation.cid)
|
|
129
131
|
await upsertOperations({
|
|
130
132
|
localDb,
|
|
131
133
|
operations: operations.map((operation) => ({
|
|
@@ -158,7 +160,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
|
|
|
158
160
|
*/
|
|
159
161
|
export async function runAddPieces({
|
|
160
162
|
localDb,
|
|
161
|
-
|
|
163
|
+
api,
|
|
162
164
|
repair,
|
|
163
165
|
concurrency,
|
|
164
166
|
batchSize,
|
|
@@ -188,6 +190,14 @@ export async function runAddPieces({
|
|
|
188
190
|
limit: 1,
|
|
189
191
|
})
|
|
190
192
|
|
|
193
|
+
// Repairs and second copies dedupe into one shared dataset; read its CIDs once instead of per batch.
|
|
194
|
+
let targetCids: Set<string> | undefined
|
|
195
|
+
if (repair.repairDataSetId == null && totalOperations > 0) {
|
|
196
|
+
const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
|
|
197
|
+
const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false })
|
|
198
|
+
targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])))
|
|
199
|
+
}
|
|
200
|
+
|
|
191
201
|
async function getNextBatch(): Promise<OperationSelect[] | null> {
|
|
192
202
|
const operations = await localDb.query.operations.findMany({
|
|
193
203
|
where: and(
|
|
@@ -207,7 +217,7 @@ export async function runAddPieces({
|
|
|
207
217
|
return operations
|
|
208
218
|
}
|
|
209
219
|
|
|
210
|
-
const addPiecesJob = createAddPiecesWorker({ localDb,
|
|
220
|
+
const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log })
|
|
211
221
|
const queue = new PQueue({ concurrency })
|
|
212
222
|
|
|
213
223
|
while (true) {
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import * as p from '@clack/prompts'
|
|
2
2
|
import * as SP from '@filoz/synapse-core/sp'
|
|
3
3
|
import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
|
|
4
|
-
import { eq } from 'drizzle-orm'
|
|
5
4
|
import type { Address } from 'viem'
|
|
5
|
+
import { getDataSet, toStringMetadata } from '../api.ts'
|
|
6
6
|
import { findRepairDataset } from '../db/find-repair-dataset.ts'
|
|
7
7
|
import { repairUpdate } from '../db/repair-update.ts'
|
|
8
8
|
import type { RepairSelect } from '../local-schema.ts'
|
|
9
|
-
import type {
|
|
9
|
+
import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
|
|
10
10
|
import { hashLink } from '../utils.ts'
|
|
11
11
|
|
|
12
12
|
export type EnsureRepairDatasetOptions = {
|
|
13
13
|
source: string
|
|
14
14
|
localDb: LocalDatabase
|
|
15
|
-
|
|
15
|
+
api: IndexerApi
|
|
16
16
|
client: WalletClient
|
|
17
17
|
repair: RepairSelect
|
|
18
18
|
payer: Address
|
|
@@ -23,14 +23,7 @@ export type EnsureRepairDatasetOptions = {
|
|
|
23
23
|
*
|
|
24
24
|
* @param options - The options for ensuring the repair dataset.
|
|
25
25
|
*/
|
|
26
|
-
export async function ensureRepairDataset({
|
|
27
|
-
source,
|
|
28
|
-
localDb,
|
|
29
|
-
indexerDb,
|
|
30
|
-
client,
|
|
31
|
-
repair,
|
|
32
|
-
payer,
|
|
33
|
-
}: EnsureRepairDatasetOptions) {
|
|
26
|
+
export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
|
|
34
27
|
const log = p.taskLog({
|
|
35
28
|
title: 'Ensuring repair dataset',
|
|
36
29
|
})
|
|
@@ -43,7 +36,7 @@ export async function ensureRepairDataset({
|
|
|
43
36
|
let datasetId: bigint | null = null
|
|
44
37
|
// check if dataset already exists
|
|
45
38
|
const existingDatasetId = await findRepairDataset({
|
|
46
|
-
|
|
39
|
+
api,
|
|
47
40
|
providerId: repair.targetProviderId,
|
|
48
41
|
payer,
|
|
49
42
|
source,
|
|
@@ -83,13 +76,7 @@ export async function ensureRepairDataset({
|
|
|
83
76
|
*
|
|
84
77
|
* @param options - The options for ensuring the replication dataset.
|
|
85
78
|
*/
|
|
86
|
-
export async function ensureReplicateDataset({
|
|
87
|
-
localDb,
|
|
88
|
-
indexerDb,
|
|
89
|
-
client,
|
|
90
|
-
repair,
|
|
91
|
-
payer,
|
|
92
|
-
}: EnsureRepairDatasetOptions) {
|
|
79
|
+
export async function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
|
|
93
80
|
const log = p.taskLog({
|
|
94
81
|
title: 'Ensuring replication dataset',
|
|
95
82
|
})
|
|
@@ -109,14 +96,7 @@ export async function ensureReplicateDataset({
|
|
|
109
96
|
return repair.targetDataSetId
|
|
110
97
|
}
|
|
111
98
|
|
|
112
|
-
const
|
|
113
|
-
const sourceDataSet = await indexerDb.query.dataSets.findFirst({
|
|
114
|
-
where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
|
|
115
|
-
columns: {
|
|
116
|
-
metadata: true,
|
|
117
|
-
withCdn: true,
|
|
118
|
-
},
|
|
119
|
-
})
|
|
99
|
+
const sourceDataSet = await getDataSet(api, repair.repairDataSetId)
|
|
120
100
|
|
|
121
101
|
if (!sourceDataSet) {
|
|
122
102
|
throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
|
|
@@ -126,8 +106,8 @@ export async function ensureReplicateDataset({
|
|
|
126
106
|
payee: provider.payee,
|
|
127
107
|
serviceURL: provider.pdp.serviceURL,
|
|
128
108
|
payer,
|
|
129
|
-
cdn: sourceDataSet.withCdn,
|
|
130
|
-
metadata: sourceDataSet.metadata
|
|
109
|
+
cdn: sourceDataSet.withCdn ?? false,
|
|
110
|
+
metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
|
|
131
111
|
})
|
|
132
112
|
log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
|
|
133
113
|
const waitForResult = await SP.waitForCreateDataSet({
|
package/src/types.ts
CHANGED
|
@@ -1,25 +1,27 @@
|
|
|
1
|
+
import type { ApiClient } from '@filecoin-project/api-client'
|
|
1
2
|
import type { Chain } from '@filoz/synapse-core/chains'
|
|
2
3
|
import type { Client } from '@libsql/client'
|
|
3
4
|
import type { LibSQLDatabase } from 'drizzle-orm/libsql'
|
|
4
|
-
import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
|
|
5
5
|
import type { z } from 'incur'
|
|
6
|
-
import type { Pool } from 'pg'
|
|
7
6
|
import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem'
|
|
8
|
-
import type
|
|
7
|
+
import type { Network } from './api.ts'
|
|
9
8
|
import type * as localSchema from './local-schema.ts'
|
|
10
9
|
import type { contextSchema } from './middleware.ts'
|
|
11
10
|
export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
|
|
12
11
|
$client: Client
|
|
13
12
|
}
|
|
14
13
|
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
/**
|
|
15
|
+
* filecoin-api client bound to the configured network.
|
|
16
|
+
*/
|
|
17
|
+
export type IndexerApi = {
|
|
18
|
+
client: ApiClient
|
|
19
|
+
network: Network
|
|
17
20
|
}
|
|
18
21
|
|
|
19
22
|
export interface Config {
|
|
20
23
|
privateKey: Hex
|
|
21
|
-
|
|
22
|
-
indexerCalibrationUrl: string
|
|
24
|
+
apiUrl: string
|
|
23
25
|
chainId: number
|
|
24
26
|
dbPath: string
|
|
25
27
|
}
|
package/src/utils.ts
CHANGED
|
@@ -3,7 +3,6 @@ import * as Piece from '@filoz/synapse-core/piece'
|
|
|
3
3
|
import type * as SP from '@filoz/synapse-core/sp'
|
|
4
4
|
import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
|
|
5
5
|
import { drizzle } from 'drizzle-orm/libsql'
|
|
6
|
-
import type { PgTable } from 'drizzle-orm/pg-core'
|
|
7
6
|
import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
|
|
8
7
|
import { z } from 'incur'
|
|
9
8
|
import { Conf } from 'iso-conf'
|
|
@@ -20,8 +19,7 @@ import type { LocalDatabase, WalletClient } from './types.ts'
|
|
|
20
19
|
|
|
21
20
|
export const configSchema = z.object({
|
|
22
21
|
privateKey: z.string().optional(),
|
|
23
|
-
|
|
24
|
-
indexerCalibrationUrl: z.url().optional(),
|
|
22
|
+
apiUrl: z.url().optional(),
|
|
25
23
|
chainId: z.number().optional(),
|
|
26
24
|
dbPath: z.string().optional(),
|
|
27
25
|
source: z.string().optional(),
|
|
@@ -81,6 +79,7 @@ export async function createLocalDatabase(dbPath: string): Promise<LocalDatabase
|
|
|
81
79
|
}) as LocalDatabase
|
|
82
80
|
|
|
83
81
|
await localDb.$client.execute('PRAGMA journal_mode = WAL')
|
|
82
|
+
await migrateLocalDatabase(localDb)
|
|
84
83
|
|
|
85
84
|
return localDb
|
|
86
85
|
}
|
|
@@ -109,6 +108,25 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
|
|
|
109
108
|
} catch {
|
|
110
109
|
// Column already exists on databases created with the updated schema.
|
|
111
110
|
}
|
|
111
|
+
try {
|
|
112
|
+
await db.$client.execute(`
|
|
113
|
+
ALTER TABLE repairs ADD COLUMN kind text NOT NULL DEFAULT 'repair'
|
|
114
|
+
`)
|
|
115
|
+
} catch {
|
|
116
|
+
// Column already exists on databases created with the updated schema.
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
await db.$client.execute(`
|
|
120
|
+
ALTER TABLE repairs ADD COLUMN source_provider_url text
|
|
121
|
+
`)
|
|
122
|
+
} catch {
|
|
123
|
+
// Column already exists on databases created with the updated schema.
|
|
124
|
+
}
|
|
125
|
+
await db.$client.execute(`
|
|
126
|
+
UPDATE repairs
|
|
127
|
+
SET kind = 'replicate'
|
|
128
|
+
WHERE repair_data_set_id IS NOT NULL AND (kind IS NULL OR kind = 'repair')
|
|
129
|
+
`)
|
|
112
130
|
await db.$client.execute(`
|
|
113
131
|
CREATE TABLE IF NOT EXISTS operations (
|
|
114
132
|
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
@@ -253,7 +271,7 @@ export async function findPieceOnProviders(providers: string[], pieceCid: string
|
|
|
253
271
|
return result
|
|
254
272
|
}
|
|
255
273
|
|
|
256
|
-
export const buildConflictUpdateColumns = <T extends
|
|
274
|
+
export const buildConflictUpdateColumns = <T extends SQLiteTable, Q extends keyof T['_']['columns']>(
|
|
257
275
|
table: T,
|
|
258
276
|
columns?: Q[]
|
|
259
277
|
) => {
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { IndexerDatabase, RepairProvider } from '../types.ts';
|
|
2
|
-
export type FindProvidersByCidOptions = {
|
|
3
|
-
indexerDb: IndexerDatabase;
|
|
4
|
-
cids: readonly string[];
|
|
5
|
-
blockNumber: bigint;
|
|
6
|
-
};
|
|
7
|
-
export type ProvidersByCid = Record<string, RepairProvider[]>;
|
|
8
|
-
export declare function findProvidersByCid({ indexerDb, cids, blockNumber, }: FindProvidersByCidOptions): Promise<ProvidersByCid>;
|
|
9
|
-
//# sourceMappingURL=find-providers-by-cid.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"find-providers-by-cid.d.ts","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,eAAe,CAAA;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAKD,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAA;AAS7D,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACZ,EAAE,yBAAyB,GAAG,OAAO,CAAC,cAAc,CAAC,CA4CrD"}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import { and, asc, eq, inArray, isNull, lte, or } from 'drizzle-orm';
|
|
2
|
-
export async function findProvidersByCid({ indexerDb, cids, blockNumber, }) {
|
|
3
|
-
const schema = indexerDb._.fullSchema;
|
|
4
|
-
const providersByCid = Object.fromEntries(cids.map((cid) => [cid, []]));
|
|
5
|
-
if (cids.length === 0)
|
|
6
|
-
return providersByCid;
|
|
7
|
-
const filters = [
|
|
8
|
-
inArray(schema.pieces.cid, [...cids]),
|
|
9
|
-
eq(schema.dataSets.deleted, false),
|
|
10
|
-
or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
|
|
11
|
-
eq(schema.pieces.removed, false),
|
|
12
|
-
or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)),
|
|
13
|
-
];
|
|
14
|
-
const rows = await indexerDb
|
|
15
|
-
.selectDistinct({
|
|
16
|
-
cid: schema.pieces.cid,
|
|
17
|
-
providerId: schema.providers.providerId,
|
|
18
|
-
providerAddress: schema.providers.providerAddress,
|
|
19
|
-
name: schema.providers.name,
|
|
20
|
-
serviceUrl: schema.providers.serviceUrl,
|
|
21
|
-
approved: schema.providers.approved,
|
|
22
|
-
endorsed: schema.providers.endorsed,
|
|
23
|
-
})
|
|
24
|
-
.from(schema.pieces)
|
|
25
|
-
.innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
|
|
26
|
-
.innerJoin(schema.providers, eq(schema.dataSets.providerId, schema.providers.providerId))
|
|
27
|
-
.where(and(...filters))
|
|
28
|
-
.orderBy(asc(schema.pieces.cid), asc(schema.providers.providerId));
|
|
29
|
-
for (const { cid, ...provider } of rows) {
|
|
30
|
-
if (provider.providerAddress && provider.serviceUrl && provider.name) {
|
|
31
|
-
providersByCid[cid]?.push({
|
|
32
|
-
providerId: provider.providerId,
|
|
33
|
-
providerAddress: provider.providerAddress,
|
|
34
|
-
name: provider.name,
|
|
35
|
-
serviceUrl: provider.serviceUrl,
|
|
36
|
-
approved: provider.approved,
|
|
37
|
-
endorsed: provider.endorsed,
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
return providersByCid;
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=find-providers-by-cid.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"find-providers-by-cid.js","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAqBpE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACe;IAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAmB,CAAA;IACzF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE5C,MAAM,OAAO,GAAG;QACd,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtF,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;KAC7E,CAAA;IAGD,MAAM,IAAI,GAAG,MAAM,SAAS;SACzB,cAAc,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,eAAe;QACjD,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;QACnC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;KACpC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SACnB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SAClF,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACxF,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;SACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrE,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAA;AACvB,CAAC"}
|