@filoz/repair-cli 0.3.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/dist/package.json +3 -3
  2. package/dist/src/api.d.ts +35 -0
  3. package/dist/src/api.d.ts.map +1 -0
  4. package/dist/src/api.js +118 -0
  5. package/dist/src/api.js.map +1 -0
  6. package/dist/src/cli.js +2 -0
  7. package/dist/src/cli.js.map +1 -1
  8. package/dist/src/commands/datasets.d.ts +3 -5
  9. package/dist/src/commands/datasets.d.ts.map +1 -1
  10. package/dist/src/commands/datasets.js +24 -33
  11. package/dist/src/commands/datasets.js.map +1 -1
  12. package/dist/src/commands/providers.d.ts +3 -5
  13. package/dist/src/commands/providers.d.ts.map +1 -1
  14. package/dist/src/commands/providers.js +32 -33
  15. package/dist/src/commands/providers.js.map +1 -1
  16. package/dist/src/commands/repair.d.ts +3 -5
  17. package/dist/src/commands/repair.d.ts.map +1 -1
  18. package/dist/src/commands/repair.js +3 -3
  19. package/dist/src/commands/repair.js.map +1 -1
  20. package/dist/src/commands/replicate.d.ts +3 -5
  21. package/dist/src/commands/replicate.d.ts.map +1 -1
  22. package/dist/src/commands/replicate.js +3 -3
  23. package/dist/src/commands/replicate.js.map +1 -1
  24. package/dist/src/commands/second-copy.d.ts +23 -0
  25. package/dist/src/commands/second-copy.d.ts.map +1 -0
  26. package/dist/src/commands/second-copy.js +175 -0
  27. package/dist/src/commands/second-copy.js.map +1 -0
  28. package/dist/src/commands/session-key.d.ts +3 -5
  29. package/dist/src/commands/session-key.d.ts.map +1 -1
  30. package/dist/src/commands/setup.d.ts.map +1 -1
  31. package/dist/src/commands/setup.js +11 -32
  32. package/dist/src/commands/setup.js.map +1 -1
  33. package/dist/src/commands/wallet.d.ts +3 -5
  34. package/dist/src/commands/wallet.d.ts.map +1 -1
  35. package/dist/src/db/dedupe-cids.d.ts +6 -7
  36. package/dist/src/db/dedupe-cids.d.ts.map +1 -1
  37. package/dist/src/db/dedupe-cids.js +2 -14
  38. package/dist/src/db/dedupe-cids.js.map +1 -1
  39. package/dist/src/db/find-repair-dataset.d.ts +3 -3
  40. package/dist/src/db/find-repair-dataset.d.ts.map +1 -1
  41. package/dist/src/db/find-repair-dataset.js +16 -14
  42. package/dist/src/db/find-repair-dataset.js.map +1 -1
  43. package/dist/src/db/get-active-data-set-ids.d.ts +9 -0
  44. package/dist/src/db/get-active-data-set-ids.d.ts.map +1 -0
  45. package/dist/src/db/get-active-data-set-ids.js +9 -0
  46. package/dist/src/db/get-active-data-set-ids.js.map +1 -0
  47. package/dist/src/db/get-alternate-provider-urls.d.ts +6 -0
  48. package/dist/src/db/get-alternate-provider-urls.d.ts.map +1 -0
  49. package/dist/src/db/get-alternate-provider-urls.js +24 -0
  50. package/dist/src/db/get-alternate-provider-urls.js.map +1 -0
  51. package/dist/src/db/get-pieces.d.ts +8 -27
  52. package/dist/src/db/get-pieces.d.ts.map +1 -1
  53. package/dist/src/db/get-pieces.js +44 -97
  54. package/dist/src/db/get-pieces.js.map +1 -1
  55. package/dist/src/db/get-repair-provider.d.ts +2 -2
  56. package/dist/src/db/get-repair-provider.d.ts.map +1 -1
  57. package/dist/src/db/get-repair-provider.js +8 -20
  58. package/dist/src/db/get-repair-provider.js.map +1 -1
  59. package/dist/src/db/repair-create.d.ts.map +1 -1
  60. package/dist/src/db/repair-create.js +5 -3
  61. package/dist/src/db/repair-create.js.map +1 -1
  62. package/dist/src/db/replicate-create.d.ts.map +1 -1
  63. package/dist/src/db/replicate-create.js +15 -21
  64. package/dist/src/db/replicate-create.js.map +1 -1
  65. package/dist/src/db/second-copy-create.d.ts +10 -0
  66. package/dist/src/db/second-copy-create.d.ts.map +1 -0
  67. package/dist/src/db/second-copy-create.js +104 -0
  68. package/dist/src/db/second-copy-create.js.map +1 -0
  69. package/dist/src/local-schema.d.ts +41 -1
  70. package/dist/src/local-schema.d.ts.map +1 -1
  71. package/dist/src/local-schema.js +2 -0
  72. package/dist/src/local-schema.js.map +1 -1
  73. package/dist/src/middleware.d.ts +7 -11
  74. package/dist/src/middleware.d.ts.map +1 -1
  75. package/dist/src/middleware.js +6 -10
  76. package/dist/src/middleware.js.map +1 -1
  77. package/dist/src/pipeline/add-pieces.d.ts +3 -3
  78. package/dist/src/pipeline/add-pieces.d.ts.map +1 -1
  79. package/dist/src/pipeline/add-pieces.js +15 -7
  80. package/dist/src/pipeline/add-pieces.js.map +1 -1
  81. package/dist/src/pipeline/create-datasets.d.ts +4 -4
  82. package/dist/src/pipeline/create-datasets.d.ts.map +1 -1
  83. package/dist/src/pipeline/create-datasets.js +7 -14
  84. package/dist/src/pipeline/create-datasets.js.map +1 -1
  85. package/dist/src/types.d.ts +6 -7
  86. package/dist/src/types.d.ts.map +1 -1
  87. package/dist/src/utils.d.ts +7 -10
  88. package/dist/src/utils.d.ts.map +1 -1
  89. package/dist/src/utils.js +21 -2
  90. package/dist/src/utils.js.map +1 -1
  91. package/package.json +3 -3
  92. package/readme.md +98 -13
  93. package/src/api.ts +220 -0
  94. package/src/cli.ts +2 -0
  95. package/src/commands/datasets.ts +24 -36
  96. package/src/commands/providers.ts +34 -42
  97. package/src/commands/repair.ts +3 -3
  98. package/src/commands/replicate.ts +3 -3
  99. package/src/commands/second-copy.ts +187 -0
  100. package/src/commands/setup.ts +12 -35
  101. package/src/db/dedupe-cids.ts +7 -24
  102. package/src/db/find-repair-dataset.ts +17 -28
  103. package/src/db/get-active-data-set-ids.ts +29 -0
  104. package/src/db/get-alternate-provider-urls.ts +43 -0
  105. package/src/db/get-pieces.ts +82 -214
  106. package/src/db/get-repair-provider.ts +10 -30
  107. package/src/db/repair-create.ts +5 -3
  108. package/src/db/replicate-create.ts +15 -21
  109. package/src/db/second-copy-create.ts +147 -0
  110. package/src/local-schema.ts +3 -0
  111. package/src/middleware.ts +7 -11
  112. package/src/pipeline/add-pieces.ts +21 -11
  113. package/src/pipeline/create-datasets.ts +9 -29
  114. package/src/types.ts +9 -7
  115. package/src/utils.ts +22 -4
  116. package/dist/src/db/find-providers-by-cid.d.ts +0 -9
  117. package/dist/src/db/find-providers-by-cid.d.ts.map +0 -1
  118. package/dist/src/db/find-providers-by-cid.js +0 -43
  119. package/dist/src/db/find-providers-by-cid.js.map +0 -1
  120. package/dist/src/indexer-schema.d.ts +0 -549
  121. package/dist/src/indexer-schema.d.ts.map +0 -1
  122. package/dist/src/indexer-schema.js +0 -56
  123. package/dist/src/indexer-schema.js.map +0 -1
  124. package/src/db/find-providers-by-cid.ts +0 -70
  125. package/src/indexer-schema.ts +0 -77
@@ -0,0 +1,147 @@
1
+ import { readFile } from 'node:fs/promises'
2
+ import { taskLog } from '@clack/prompts'
3
+ import { getBlockNumber } from 'viem/actions'
4
+ import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
5
+ import type { OperationInsert } from '../local-schema.ts'
6
+ import type { Context } from '../types.ts'
7
+ import { getRepairProvider } from './get-repair-provider.ts'
8
+
9
+ const OPERATIONS_INSERT_BATCH_SIZE = 500
10
+
11
+ export interface SecondCopyCreateOptions extends Context {
12
+ piecesFile: string
13
+ sourceProviderUrl: string
14
+ targetProviderId: bigint
15
+ }
16
+
17
+ type PiecesFile = {
18
+ aggregates: Array<{ piece_cid?: unknown }>
19
+ }
20
+
21
+ /**
22
+ * Normalize a provider service URL for storage and equality checks.
23
+ */
24
+ export function normalizeProviderUrl(url: string): string {
25
+ let parsed: URL
26
+ try {
27
+ parsed = new URL(url)
28
+ } catch {
29
+ throw new RepairCreationError(`Invalid source provider URL: ${url}`)
30
+ }
31
+
32
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
33
+ throw new RepairCreationError(`Source provider URL must be http(s): ${url}`)
34
+ }
35
+
36
+ // Drop trailing slash so endpoint comparisons are stable.
37
+ return parsed.toString().replace(/\/$/, '')
38
+ }
39
+
40
+ /**
41
+ * Parse a second-copy pieces file and return unique aggregate piece CIDs.
42
+ */
43
+ export async function parsePieceCidsFromFile(piecesFile: string): Promise<string[]> {
44
+ let raw: unknown
45
+ try {
46
+ raw = JSON.parse(await readFile(piecesFile, 'utf8'))
47
+ } catch (error) {
48
+ throw new RepairCreationError(
49
+ error instanceof Error ? `Failed to read pieces file: ${error.message}` : 'Failed to read pieces file'
50
+ )
51
+ }
52
+
53
+ if (raw == null || typeof raw !== 'object' || !Array.isArray((raw as PiecesFile).aggregates)) {
54
+ throw new RepairCreationError('Pieces file must contain an "aggregates" array')
55
+ }
56
+
57
+ const cids = new Set<string>()
58
+ for (const aggregate of (raw as PiecesFile).aggregates) {
59
+ if (typeof aggregate?.piece_cid !== 'string' || aggregate.piece_cid.length === 0) {
60
+ throw new RepairCreationError('Each aggregate must include a non-empty "piece_cid" string')
61
+ }
62
+ cids.add(aggregate.piece_cid)
63
+ }
64
+
65
+ if (cids.size === 0) {
66
+ throw new RepairCreationError('Pieces file contains no piece CIDs')
67
+ }
68
+
69
+ return [...cids]
70
+ }
71
+
72
+ /**
73
+ * Prepare a second-copy job from a pieces file and an explicit source provider URL.
74
+ *
75
+ * @param options - The options for creating a second-copy job.
76
+ * @returns The ID of the created local job.
77
+ */
78
+ export async function secondCopyCreate(options: SecondCopyCreateOptions): Promise<number> {
79
+ const { api, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
80
+ const localSchema = localDb._.fullSchema
81
+ const now = Date.now()
82
+ const blockNumber = await getBlockNumber(client)
83
+ const normalizedSourceUrl = normalizeProviderUrl(sourceProviderUrl)
84
+
85
+ const log = taskLog({
86
+ title: 'Creating second-copy',
87
+ limit: 10,
88
+ retainLog: true,
89
+ })
90
+
91
+ const targetProvider = await getRepairProvider({
92
+ api,
93
+ providerId: targetProviderId,
94
+ })
95
+
96
+ if (!targetProvider) {
97
+ throw new NoAlternateProviderError(targetProviderId)
98
+ }
99
+
100
+ const normalizedTargetUrl = normalizeProviderUrl(targetProvider.serviceUrl)
101
+ if (normalizedSourceUrl === normalizedTargetUrl) {
102
+ throw new RepairCreationError('Source provider URL must differ from the target provider URL')
103
+ }
104
+
105
+ const pieceCids = await parsePieceCidsFromFile(piecesFile)
106
+ log.message(`Loaded ${pieceCids.length} unique piece CIDs from ${piecesFile}`)
107
+
108
+ const [repair] = await localDb
109
+ .insert(localSchema.repairs)
110
+ .values({
111
+ kind: 'second_copy',
112
+ repairProviderId: 0n,
113
+ repairDataSetId: null,
114
+ sourceProviderUrl: normalizedSourceUrl,
115
+ targetProviderId: targetProvider.providerId,
116
+ targetProviderUrl: targetProvider.serviceUrl,
117
+ targetDataSetId: null,
118
+ blockNumber,
119
+ createdAt: now,
120
+ updatedAt: now,
121
+ })
122
+ .returning({ id: localSchema.repairs.id })
123
+
124
+ if (!repair) throw new RepairCreationError('Failed to create second-copy row')
125
+
126
+ let totalOperations = 0
127
+ for (let offset = 0; offset < pieceCids.length; offset += OPERATIONS_INSERT_BATCH_SIZE) {
128
+ const batchCids = pieceCids.slice(offset, offset + OPERATIONS_INSERT_BATCH_SIZE)
129
+ const operations: OperationInsert[] = batchCids.map((cid) => ({
130
+ repairId: repair.id,
131
+ type: 'add_piece',
132
+ status: 'pending',
133
+ cid,
134
+ metadata: {},
135
+ alternateProvider: normalizedSourceUrl,
136
+ createdAt: now,
137
+ updatedAt: now,
138
+ }))
139
+
140
+ await localDb.insert(localSchema.operations).values(operations)
141
+ totalOperations += operations.length
142
+ log.message(`Inserted ${operations.length} operations (${totalOperations}/${pieceCids.length})`)
143
+ }
144
+
145
+ log.success(`Created second-copy ${repair.id} with ${totalOperations} operations`, { showLog: true })
146
+ return repair.id
147
+ }
@@ -6,6 +6,7 @@ import { customType, sqliteTable as table } from 'drizzle-orm/sqlite-core'
6
6
  import * as Json from 'iso-base/json'
7
7
 
8
8
  export type RepairStatus = 'pending' | 'completed' | 'failed'
9
+ export type RepairKind = 'repair' | 'replicate' | 'second_copy'
9
10
  export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped'
10
11
  export type OperationType = 'create_dataset' | 'add_piece'
11
12
 
@@ -44,8 +45,10 @@ export type RepairUpdate = Partial<RepairInsert>
44
45
  export const repairs = table('repairs', {
45
46
  id: t.int().primaryKey({ autoIncrement: true }),
46
47
  status: t.text().$type<RepairStatus>().notNull().default('pending'),
48
+ kind: t.text().$type<RepairKind>().notNull().default('repair'),
47
49
  repairProviderId: bigintType('repair_provider_id').notNull(),
48
50
  repairDataSetId: bigintType('repair_data_set_id'),
51
+ sourceProviderUrl: t.text('source_provider_url'),
49
52
  targetProviderId: bigintType('target_provider_id').notNull(),
50
53
  targetProviderUrl: t.text('target_provider_url').notNull(),
51
54
  targetDataSetId: bigintType('target_data_set_id'),
package/src/middleware.ts CHANGED
@@ -1,13 +1,12 @@
1
1
  import type { Chain } from '@filoz/synapse-core/chains'
2
- import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres'
3
2
  import { middleware, z } from 'incur'
4
3
  import type { Account, Client, Transport } from 'viem'
5
- import * as indexerSchema from './indexer-schema.ts'
6
- import type { IndexerDatabase, LocalDatabase } from './types.ts'
4
+ import { createIndexerApi } from './api.ts'
5
+ import type { IndexerApi, LocalDatabase } from './types.ts'
7
6
  import { config, createLocalDatabase, getClient } from './utils.ts'
8
7
 
9
8
  export const contextSchema = z.object({
10
- indexerDb: z.custom<IndexerDatabase>(),
9
+ api: z.custom<IndexerApi>(),
11
10
  localDb: z.custom<LocalDatabase>(),
12
11
  config: z.custom<typeof config>(),
13
12
  client: z.custom<Client<Transport, Chain, Account>>(),
@@ -17,9 +16,9 @@ export const contextSchema = z.object({
17
16
  })
18
17
 
19
18
  export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
20
- const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl, source } = config.store
19
+ const { dbPath, chainId, apiUrl, source } = config.store
21
20
 
22
- if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl || !source) {
21
+ if (!dbPath || !chainId || !source) {
23
22
  return c.error({
24
23
  code: 'CONFIG_NOT_SET',
25
24
  message: 'Config not set. Please run `repair setup` first.',
@@ -28,13 +27,11 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
28
27
  }
29
28
 
30
29
  const localDb = await createLocalDatabase(dbPath)
31
- const indexerDb = drizzlePostgres(chainId === 314 ? indexerMainnetUrl : indexerCalibrationUrl, {
32
- schema: indexerSchema,
33
- })
30
+ const api = createIndexerApi({ baseUrl: apiUrl, chainId })
34
31
 
35
32
  const { client, chain } = getClient(chainId)
36
33
  c.set('localDb', localDb)
37
- c.set('indexerDb', indexerDb)
34
+ c.set('api', api)
38
35
  c.set('config', config)
39
36
  c.set('client', client)
40
37
  c.set('chain', chain)
@@ -43,5 +40,4 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
43
40
  await next()
44
41
 
45
42
  localDb.$client.close()
46
- await indexerDb.$client.end()
47
43
  })
@@ -3,17 +3,18 @@ import * as Piece from '@filoz/synapse-core/piece'
3
3
  import * as SP from '@filoz/synapse-core/sp'
4
4
  import { and, asc, eq, gt, inArray } from 'drizzle-orm'
5
5
  import PQueue from 'p-queue'
6
+ import { listDataSetPieces } from '../api.ts'
6
7
  import { dedupeCids } from '../db/dedupe-cids.ts'
7
8
  import { getTargetDataset } from '../db/get-target-dataset.ts'
8
9
  import { repairUpdate } from '../db/repair-update.ts'
9
10
  import { upsertOperations } from '../db/upsert-operations.ts'
10
11
  import type { OperationSelect, RepairSelect } from '../local-schema.ts'
11
- import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
12
+ import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
12
13
  import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
13
14
 
14
15
  export type RunPullPiecesPhaseOptions = {
15
16
  localDb: LocalDatabase
16
- indexerDb: IndexerDatabase
17
+ api: IndexerApi
17
18
  repair: RepairSelect
18
19
  concurrency: number
19
20
  batchSize: number
@@ -22,7 +23,8 @@ export type RunPullPiecesPhaseOptions = {
22
23
 
23
24
  type CreateAddPiecesWorkerOptions = {
24
25
  localDb: LocalDatabase
25
- indexerDb: IndexerDatabase
26
+ /** CIDs already in the target dataset; repairs only. Updated as batches are added. */
27
+ targetCids?: Set<string>
26
28
  repair: RepairSelect
27
29
  client: WalletClient
28
30
  state: {
@@ -42,12 +44,11 @@ type AddPiecesJobOptions = {
42
44
  /**
43
45
  * Create a worker function that adds pieces to the target repair dataset.
44
46
  */
45
- function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
47
+ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
46
48
  return async (options: AddPiecesJobOptions) => {
47
49
  let completedOps = 0
48
50
  let failedOps = 0
49
51
  let operations: OperationSelect[] = options.operations
50
- const isRepair = repair.repairDataSetId == null
51
52
 
52
53
  const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
53
54
 
@@ -55,8 +56,8 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
55
56
  const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
56
57
 
57
58
  // dedupe operations by CID on the target dataset for repairs jobs
58
- if (isRepair) {
59
- operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
59
+ if (targetCids) {
60
+ operations = await dedupeCids({ localDb, existingCids: targetCids, operations })
60
61
  }
61
62
  // A previous run may have submitted a transaction and crashed before marking rows completed.
62
63
  const operationsBeforeConfirmationCheck = operations.length
@@ -75,7 +76,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
75
76
  dataSetId: dataset.dataSetId,
76
77
  clientDataSetId: dataset.clientDataSetId,
77
78
  pieces: operationsToPullPieces(operations),
78
- timeout: 1000 * 60 * 30,
79
+ timeout: 1000 * 60 * 30, // 30 minutes
79
80
  onStatus: (status) => {
80
81
  const completed = status.pieces.filter((piece) => piece.status === 'complete').length
81
82
  const failed = status.pieces.filter((piece) => piece.status === 'failed').length
@@ -111,7 +112,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
111
112
  clientDataSetId: dataset.clientDataSetId,
112
113
  pieces: operations.map((operation) => ({
113
114
  pieceCid: Piece.from(operation.cid),
114
- metadata: isRepair ? undefined : operation.metadata,
115
+ metadata: targetCids ? undefined : operation.metadata,
115
116
  })),
116
117
  })
117
118
  await upsertOperations({
@@ -126,6 +127,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
126
127
  const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
127
128
  state.completedOperations += operations.length
128
129
  completedOps += operations.length
130
+ for (const operation of operations) targetCids?.add(operation.cid)
129
131
  await upsertOperations({
130
132
  localDb,
131
133
  operations: operations.map((operation) => ({
@@ -158,7 +160,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
158
160
  */
159
161
  export async function runAddPieces({
160
162
  localDb,
161
- indexerDb,
163
+ api,
162
164
  repair,
163
165
  concurrency,
164
166
  batchSize,
@@ -188,6 +190,14 @@ export async function runAddPieces({
188
190
  limit: 1,
189
191
  })
190
192
 
193
+ // Repairs and second copies dedupe into one shared dataset; read its CIDs once instead of per batch.
194
+ let targetCids: Set<string> | undefined
195
+ if (repair.repairDataSetId == null && totalOperations > 0) {
196
+ const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
197
+ const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false })
198
+ targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])))
199
+ }
200
+
191
201
  async function getNextBatch(): Promise<OperationSelect[] | null> {
192
202
  const operations = await localDb.query.operations.findMany({
193
203
  where: and(
@@ -207,7 +217,7 @@ export async function runAddPieces({
207
217
  return operations
208
218
  }
209
219
 
210
- const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log })
220
+ const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log })
211
221
  const queue = new PQueue({ concurrency })
212
222
 
213
223
  while (true) {
@@ -1,18 +1,18 @@
1
1
  import * as p from '@clack/prompts'
2
2
  import * as SP from '@filoz/synapse-core/sp'
3
3
  import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
4
- import { eq } from 'drizzle-orm'
5
4
  import type { Address } from 'viem'
5
+ import { getDataSet, toStringMetadata } from '../api.ts'
6
6
  import { findRepairDataset } from '../db/find-repair-dataset.ts'
7
7
  import { repairUpdate } from '../db/repair-update.ts'
8
8
  import type { RepairSelect } from '../local-schema.ts'
9
- import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
9
+ import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
10
10
  import { hashLink } from '../utils.ts'
11
11
 
12
12
  export type EnsureRepairDatasetOptions = {
13
13
  source: string
14
14
  localDb: LocalDatabase
15
- indexerDb: IndexerDatabase
15
+ api: IndexerApi
16
16
  client: WalletClient
17
17
  repair: RepairSelect
18
18
  payer: Address
@@ -23,14 +23,7 @@ export type EnsureRepairDatasetOptions = {
23
23
  *
24
24
  * @param options - The options for ensuring the repair dataset.
25
25
  */
26
- export async function ensureRepairDataset({
27
- source,
28
- localDb,
29
- indexerDb,
30
- client,
31
- repair,
32
- payer,
33
- }: EnsureRepairDatasetOptions) {
26
+ export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
34
27
  const log = p.taskLog({
35
28
  title: 'Ensuring repair dataset',
36
29
  })
@@ -43,7 +36,7 @@ export async function ensureRepairDataset({
43
36
  let datasetId: bigint | null = null
44
37
  // check if dataset already exists
45
38
  const existingDatasetId = await findRepairDataset({
46
- indexerDb,
39
+ api,
47
40
  providerId: repair.targetProviderId,
48
41
  payer,
49
42
  source,
@@ -83,13 +76,7 @@ export async function ensureRepairDataset({
83
76
  *
84
77
  * @param options - The options for ensuring the replication dataset.
85
78
  */
86
- export async function ensureReplicateDataset({
87
- localDb,
88
- indexerDb,
89
- client,
90
- repair,
91
- payer,
92
- }: EnsureRepairDatasetOptions) {
79
+ export async function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
93
80
  const log = p.taskLog({
94
81
  title: 'Ensuring replication dataset',
95
82
  })
@@ -109,14 +96,7 @@ export async function ensureReplicateDataset({
109
96
  return repair.targetDataSetId
110
97
  }
111
98
 
112
- const schema = indexerDb._.fullSchema
113
- const sourceDataSet = await indexerDb.query.dataSets.findFirst({
114
- where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
115
- columns: {
116
- metadata: true,
117
- withCdn: true,
118
- },
119
- })
99
+ const sourceDataSet = await getDataSet(api, repair.repairDataSetId)
120
100
 
121
101
  if (!sourceDataSet) {
122
102
  throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
@@ -126,8 +106,8 @@ export async function ensureReplicateDataset({
126
106
  payee: provider.payee,
127
107
  serviceURL: provider.pdp.serviceURL,
128
108
  payer,
129
- cdn: sourceDataSet.withCdn,
130
- metadata: sourceDataSet.metadata ?? undefined,
109
+ cdn: sourceDataSet.withCdn ?? false,
110
+ metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
131
111
  })
132
112
  log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
133
113
  const waitForResult = await SP.waitForCreateDataSet({
package/src/types.ts CHANGED
@@ -1,25 +1,27 @@
1
+ import type { ApiClient } from '@filecoin-project/api-client'
1
2
  import type { Chain } from '@filoz/synapse-core/chains'
2
3
  import type { Client } from '@libsql/client'
3
4
  import type { LibSQLDatabase } from 'drizzle-orm/libsql'
4
- import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
5
5
  import type { z } from 'incur'
6
- import type { Pool } from 'pg'
7
6
  import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem'
8
- import type * as indexerSchema from './indexer-schema.ts'
7
+ import type { Network } from './api.ts'
9
8
  import type * as localSchema from './local-schema.ts'
10
9
  import type { contextSchema } from './middleware.ts'
11
10
  export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
12
11
  $client: Client
13
12
  }
14
13
 
15
- export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
16
- $client: Pool
14
+ /**
15
+ * filecoin-api client bound to the configured network.
16
+ */
17
+ export type IndexerApi = {
18
+ client: ApiClient
19
+ network: Network
17
20
  }
18
21
 
19
22
  export interface Config {
20
23
  privateKey: Hex
21
- indexerMainnetUrl: string
22
- indexerCalibrationUrl: string
24
+ apiUrl: string
23
25
  chainId: number
24
26
  dbPath: string
25
27
  }
package/src/utils.ts CHANGED
@@ -3,7 +3,6 @@ import * as Piece from '@filoz/synapse-core/piece'
3
3
  import type * as SP from '@filoz/synapse-core/sp'
4
4
  import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
5
5
  import { drizzle } from 'drizzle-orm/libsql'
6
- import type { PgTable } from 'drizzle-orm/pg-core'
7
6
  import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
8
7
  import { z } from 'incur'
9
8
  import { Conf } from 'iso-conf'
@@ -20,8 +19,7 @@ import type { LocalDatabase, WalletClient } from './types.ts'
20
19
 
21
20
  export const configSchema = z.object({
22
21
  privateKey: z.string().optional(),
23
- indexerMainnetUrl: z.url().optional(),
24
- indexerCalibrationUrl: z.url().optional(),
22
+ apiUrl: z.url().optional(),
25
23
  chainId: z.number().optional(),
26
24
  dbPath: z.string().optional(),
27
25
  source: z.string().optional(),
@@ -81,6 +79,7 @@ export async function createLocalDatabase(dbPath: string): Promise<LocalDatabase
81
79
  }) as LocalDatabase
82
80
 
83
81
  await localDb.$client.execute('PRAGMA journal_mode = WAL')
82
+ await migrateLocalDatabase(localDb)
84
83
 
85
84
  return localDb
86
85
  }
@@ -109,6 +108,25 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
109
108
  } catch {
110
109
  // Column already exists on databases created with the updated schema.
111
110
  }
111
+ try {
112
+ await db.$client.execute(`
113
+ ALTER TABLE repairs ADD COLUMN kind text NOT NULL DEFAULT 'repair'
114
+ `)
115
+ } catch {
116
+ // Column already exists on databases created with the updated schema.
117
+ }
118
+ try {
119
+ await db.$client.execute(`
120
+ ALTER TABLE repairs ADD COLUMN source_provider_url text
121
+ `)
122
+ } catch {
123
+ // Column already exists on databases created with the updated schema.
124
+ }
125
+ await db.$client.execute(`
126
+ UPDATE repairs
127
+ SET kind = 'replicate'
128
+ WHERE repair_data_set_id IS NOT NULL AND (kind IS NULL OR kind = 'repair')
129
+ `)
112
130
  await db.$client.execute(`
113
131
  CREATE TABLE IF NOT EXISTS operations (
114
132
  id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
@@ -253,7 +271,7 @@ export async function findPieceOnProviders(providers: string[], pieceCid: string
253
271
  return result
254
272
  }
255
273
 
256
- export const buildConflictUpdateColumns = <T extends PgTable | SQLiteTable, Q extends keyof T['_']['columns']>(
274
+ export const buildConflictUpdateColumns = <T extends SQLiteTable, Q extends keyof T['_']['columns']>(
257
275
  table: T,
258
276
  columns?: Q[]
259
277
  ) => {
@@ -1,9 +0,0 @@
1
- import type { IndexerDatabase, RepairProvider } from '../types.ts';
2
- export type FindProvidersByCidOptions = {
3
- indexerDb: IndexerDatabase;
4
- cids: readonly string[];
5
- blockNumber: bigint;
6
- };
7
- export type ProvidersByCid = Record<string, RepairProvider[]>;
8
- export declare function findProvidersByCid({ indexerDb, cids, blockNumber, }: FindProvidersByCidOptions): Promise<ProvidersByCid>;
9
- //# sourceMappingURL=find-providers-by-cid.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"find-providers-by-cid.d.ts","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,eAAe,CAAA;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAKD,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAA;AAS7D,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACZ,EAAE,yBAAyB,GAAG,OAAO,CAAC,cAAc,CAAC,CA4CrD"}
@@ -1,43 +0,0 @@
1
- import { and, asc, eq, inArray, isNull, lte, or } from 'drizzle-orm';
2
- export async function findProvidersByCid({ indexerDb, cids, blockNumber, }) {
3
- const schema = indexerDb._.fullSchema;
4
- const providersByCid = Object.fromEntries(cids.map((cid) => [cid, []]));
5
- if (cids.length === 0)
6
- return providersByCid;
7
- const filters = [
8
- inArray(schema.pieces.cid, [...cids]),
9
- eq(schema.dataSets.deleted, false),
10
- or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
11
- eq(schema.pieces.removed, false),
12
- or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)),
13
- ];
14
- const rows = await indexerDb
15
- .selectDistinct({
16
- cid: schema.pieces.cid,
17
- providerId: schema.providers.providerId,
18
- providerAddress: schema.providers.providerAddress,
19
- name: schema.providers.name,
20
- serviceUrl: schema.providers.serviceUrl,
21
- approved: schema.providers.approved,
22
- endorsed: schema.providers.endorsed,
23
- })
24
- .from(schema.pieces)
25
- .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
26
- .innerJoin(schema.providers, eq(schema.dataSets.providerId, schema.providers.providerId))
27
- .where(and(...filters))
28
- .orderBy(asc(schema.pieces.cid), asc(schema.providers.providerId));
29
- for (const { cid, ...provider } of rows) {
30
- if (provider.providerAddress && provider.serviceUrl && provider.name) {
31
- providersByCid[cid]?.push({
32
- providerId: provider.providerId,
33
- providerAddress: provider.providerAddress,
34
- name: provider.name,
35
- serviceUrl: provider.serviceUrl,
36
- approved: provider.approved,
37
- endorsed: provider.endorsed,
38
- });
39
- }
40
- }
41
- return providersByCid;
42
- }
43
- //# sourceMappingURL=find-providers-by-cid.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"find-providers-by-cid.js","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAqBpE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACe;IAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAmB,CAAA;IACzF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE5C,MAAM,OAAO,GAAG;QACd,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtF,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;KAC7E,CAAA;IAGD,MAAM,IAAI,GAAG,MAAM,SAAS;SACzB,cAAc,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,eAAe;QACjD,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;QACnC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;KACpC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SACnB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SAClF,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACxF,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;SACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrE,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAA;AACvB,CAAC"}