@filoz/repair-cli 0.1.2 → 0.2.0

package/src/db/get-pieces.ts

@@ -3,7 +3,7 @@ import { and, asc, eq, isNull, lte, or } from 'drizzle-orm'
 import pMap from 'p-map'
 import type { OperationInsert } from '../local-schema.ts'
 import type { IndexerDatabase } from '../types.ts'
-import { getProvidersByCid } from './get-providers-by-cid.ts'
+import { findProvidersByCid } from './find-providers-by-cid.ts'
 
 /** Default page size when paginating pieces from the indexer. */
 export const DEFAULT_PIECES_PAGE_SIZE = 3000
@@ -43,6 +43,34 @@ export type GetPiecesPageResult = {
 /** Options for {@link forEachPiecesPage}; pagination state is managed internally. */
 export type ForEachPiecesPageOptions = Omit<GetPiecesPageOptions, 'offset' | 'seenCids'>
 
+/** Options for fetching one page of source-dataset pieces for replication. */
+export type GetDataSetPiecesPageOptions = {
+  indexerDb: IndexerDatabase
+  /** Source dataset whose pieces are being replicated. */
+  dataSetId: bigint
+  /** Local job row to attach operations to. */
+  repairId: number
+  /** Provider URL that serves the source dataset pieces. */
+  sourceProviderUrl: string
+  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
+  limit?: number
+  /** SQL offset for the indexer query. */
+  offset?: number
+}
+
+/** Result of a single {@link getDataSetPiecesPage} call. */
+export type GetDataSetPiecesPageResult = {
+  /** `add_piece` operations ready to insert for this page. */
+  operations: OperationInsert[]
+  /** Whether another indexer page may exist after this one. */
+  hasMore: boolean
+  /** Offset to pass as `offset` on the next page. */
+  nextOffset: number
+}
+
+/** Options for {@link forEachDataSetPiecesPage}; pagination state is managed internally. */
+export type ForEachDataSetPiecesPageOptions = Omit<GetDataSetPiecesPageOptions, 'offset'>
+
 type PieceForOperation = {
   cid: string
   metadata: Record<string, string> | null
@@ -107,10 +135,9 @@ export async function getPiecesPage({
   }
 
   // Resolve pull sources in one query per page; exclude the provider being repaired from alternates.
-  const providersByCid = await getProvidersByCid({
+  const providersByCid = await findProvidersByCid({
     indexerDb,
     cids: pieces.map((piece) => piece.cid),
-    excludedProviderIds: [],
     blockNumber,
   })
 
@@ -157,6 +184,53 @@ export async function getPiecesPage({
   }
 }
 
+/**
+ * Fetch one page of pieces for a specific dataset and map them to replication operations.
+ *
+ * Unlike repairs, replication preserves source dataset ordering and does not dedupe repeated CIDs.
+ *
+ * @param options - Indexer connection, dataset context, and optional pagination state.
+ * @returns Operations for this page plus pagination cursors.
+ */
+export async function getDataSetPiecesPage({
+  indexerDb,
+  dataSetId,
+  repairId,
+  sourceProviderUrl,
+  limit = DEFAULT_PIECES_PAGE_SIZE,
+  offset = 0,
+}: GetDataSetPiecesPageOptions): Promise<GetDataSetPiecesPageResult> {
+  const schema = indexerDb._.fullSchema
+  const rows = await indexerDb
+    .select({
+      cid: schema.pieces.cid,
+      metadata: schema.pieces.metadata,
+    })
+    .from(schema.pieces)
+    .where(and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false)))
+    .orderBy(asc(schema.pieces.pieceId))
+    .limit(limit)
+    .offset(offset)
+
+  const now = Date.now()
+  const operations: OperationInsert[] = rows.map(({ cid, metadata }) => ({
+    repairId,
+    type: 'add_piece',
+    status: 'pending',
+    cid,
+    metadata: metadata ?? {},
+    alternateProvider: sourceProviderUrl,
+    createdAt: now,
+    updatedAt: now,
+  }))
+
+  return {
+    operations,
+    hasMore: rows.length === limit,
+    nextOffset: offset + rows.length,
+  }
+}
+
 /**
  * Walk every page of `add_piece` operations for a provider, invoking `onPage` per batch.
  *
@@ -187,3 +261,31 @@ export async function forEachPiecesPage(
     hasMore = page.hasMore
   }
 }
+
+/**
+ * Walk every page of `add_piece` operations for a source dataset, invoking `onPage` per batch.
+ *
+ * Replication pagination intentionally has no CID dedupe state so duplicate pieces are preserved.
+ *
+ * @param options - Same inputs as {@link getDataSetPiecesPage} except pagination cursor.
+ * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ */
+export async function forEachDataSetPiecesPage(
+  options: ForEachDataSetPiecesPageOptions,
+  onPage: (page: GetDataSetPiecesPageResult) => Promise<void>
+): Promise<void> {
+  let offset = 0
+  let hasMore = true
+
+  while (hasMore) {
+    const page = await getDataSetPiecesPage({
+      ...options,
+      offset,
+    })
+
+    await onPage(page)
+
+    offset = page.nextOffset
+    hasMore = page.hasMore
+  }
+}

package/src/db/get-target-dataset.ts

@@ -6,7 +6,7 @@ import type { LocalDatabase, WalletClient } from '../types.ts'
 const targetDatasetCache = new Map<number, getDataSet.OutputType>()
 
 /**
- * Get the single IPFS-enabled target dataset for a repair.
+ * Get a targetdataset for a repair.
  *
  * @param options - The options for getting the target dataset.
  */

package/src/db/repair-create.ts

@@ -47,6 +47,7 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
     .insert(localSchema.repairs)
     .values({
       repairProviderId,
+      repairDataSetId: null,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
       targetDataSetId: null,

package/src/db/repair-delete.ts

@@ -19,11 +19,6 @@ export async function repairDelete({ localDb, repairId }: RepairDeleteOptions):
   const repair = await localDb.query.repairs.findFirst({
     where: eq(localSchema.repairs.id, repairId),
     columns: { id: true },
-    with: {
-      operations: {
-        columns: { id: true },
-      },
-    },
   })
 
   if (!repair) {

package/src/db/replicate-create.ts

@@ -0,0 +1,106 @@
+import { taskLog } from '@clack/prompts'
+import { eq } from 'drizzle-orm'
+import { getBlockNumber } from 'viem/actions'
+import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
+import type { Context } from '../types.ts'
+import { forEachDataSetPiecesPage } from './get-pieces.ts'
+import { getRepairProvider } from './get-repair-provider.ts'
+
+export interface ReplicateCreateOptions extends Context {
+  dataSetId: bigint
+  targetProviderId: bigint
+}
+
+/**
+ * Prepare a replication job by creating a local row and inserting every source dataset piece.
+ *
+ * @param options - The options for creating a replication job.
+ * @returns The ID of the created local job.
+ */
+export async function replicateCreate(options: ReplicateCreateOptions): Promise<number> {
+  const { indexerDb, localDb, dataSetId, targetProviderId, client } = options
+  const indexerSchema = indexerDb._.fullSchema
+  const localSchema = localDb._.fullSchema
+  const now = Date.now()
+  const blockNumber = await getBlockNumber(client)
+
+  const log = taskLog({
+    title: 'Creating replication',
+    limit: 10,
+    retainLog: true,
+  })
+
+  const [sourceDataSet] = await indexerDb
+    .select({
+      dataSetId: indexerSchema.dataSets.dataSetId,
+      providerId: indexerSchema.dataSets.providerId,
+      deleted: indexerSchema.dataSets.deleted,
+    })
+    .from(indexerSchema.dataSets)
+    .where(eq(indexerSchema.dataSets.dataSetId, dataSetId))
+    .limit(1)
+
+  if (!sourceDataSet || sourceDataSet.deleted) {
+    throw new RepairCreationError(`Source dataset ${dataSetId} not found or deleted`)
+  }
+
+  if (targetProviderId === sourceDataSet.providerId) {
+    throw new RepairCreationError('Target provider must differ from the source dataset provider')
+  }
+
+  const sourceProvider = await getRepairProvider({
+    indexerDb,
+    providerId: sourceDataSet.providerId,
+  })
+
+  if (!sourceProvider) {
+    throw new RepairCreationError(`Source provider ${sourceDataSet.providerId} not found or inactive`)
+  }
+
+  const targetProvider = await getRepairProvider({
+    indexerDb,
+    providerId: targetProviderId,
+  })
+
+  if (!targetProvider) {
+    throw new NoAlternateProviderError(targetProviderId)
+  }
+
+  const [repair] = await localDb
+    .insert(localSchema.repairs)
+    .values({
+      repairProviderId: sourceProvider.providerId,
+      repairDataSetId: sourceDataSet.dataSetId,
+      targetProviderId: targetProvider.providerId,
+      targetProviderUrl: targetProvider.serviceUrl,
+      targetDataSetId: null,
+      blockNumber,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .returning({ id: localSchema.repairs.id })
+
+  if (!repair) throw new RepairCreationError('Failed to create replication row')
+
+  let totalOperations = 0
+  await forEachDataSetPiecesPage(
+    {
+      indexerDb,
+      dataSetId: sourceDataSet.dataSetId,
+      repairId: repair.id,
+      sourceProviderUrl: sourceProvider.serviceUrl,
+    },
+    async (page) => {
+      totalOperations += page.operations.length
+
+      if (page.operations.length > 0) {
+        await localDb.insert(localSchema.operations).values(page.operations)
+      }
+
+      log.message(`Inserted ${page.operations.length} operations`)
+    }
+  )
+
+  log.success(`Created replication ${repair.id} with ${totalOperations} operations`, { showLog: true })
+  return repair.id
+}

package/src/db/upsert-operations.ts

@@ -18,6 +18,6 @@ export async function upsertOperations({ localDb, operations }: UpsertOperations
     .values(operations.map((operation) => ({ ...operation, updatedAt: now })))
     .onConflictDoUpdate({
       target: localDb._.fullSchema.operations.id,
-      set: buildConflictUpdateColumns(localSchema.operations, ['status', 'error', 'updatedAt']),
+      set: buildConflictUpdateColumns(localSchema.operations, ['status', 'error', 'updatedAt', 'result']),
     })
 }

package/src/local-schema.ts

@@ -51,6 +51,7 @@ export const repairs = table('repairs', {
   id: t.int().primaryKey({ autoIncrement: true }),
   status: t.text().$type<RepairStatus>().notNull().default('pending'),
   repairProviderId: bigintType('repair_provider_id').notNull(),
+  repairDataSetId: bigintType('repair_data_set_id'),
   targetProviderId: bigintType('target_provider_id').notNull(),
   targetProviderUrl: t.text('target_provider_url').notNull(),
   targetDataSetId: bigintType('target_data_set_id'),

package/src/middleware.ts

@@ -12,12 +12,13 @@ export const contextSchema = z.object({
   config: z.custom<typeof config>(),
   client: z.custom<Client<Transport, Chain, Account>>(),
   chain: z.custom<Chain>(),
+  source: z.string(),
 })
 
 export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
-  const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl } = config.store
+  const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl, source } = config.store
 
-  if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl) {
+  if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl || !source) {
     return c.error({
       code: 'CONFIG_NOT_SET',
       message: 'Config not set. Please run `repair setup` first.',
@@ -36,6 +37,7 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
   c.set('config', config)
   c.set('client', client)
   c.set('chain', chain)
+  c.set('source', source)
   await next()
 
   localDb.$client.close()

package/src/pipeline/add-pieces.ts

@@ -0,0 +1,214 @@
+import { taskLog } from '@clack/prompts'
+import * as Piece from '@filoz/synapse-core/piece'
+import * as SP from '@filoz/synapse-core/sp'
+import { and, asc, eq, gt, inArray } from 'drizzle-orm'
+import PQueue from 'p-queue'
+import { dedupeCids } from '../db/dedupe-cids.ts'
+import { getTargetDataset } from '../db/get-target-dataset.ts'
+import { repairUpdate } from '../db/repair-update.ts'
+import { upsertOperations } from '../db/upsert-operations.ts'
+import type { OperationSelect, RepairSelect } from '../local-schema.ts'
+import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import { excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
+
+export type RunPullPiecesPhaseOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  concurrency: number
+  batchSize: number
+  client: WalletClient
+}
+
+type CreateAddPiecesWorkerOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  client: WalletClient
+  state: {
+    totalBatches: number
+    totalOperations: number
+    completedOperations: number
+    failedOperations: number
+  }
+  log: ReturnType<typeof taskLog>
+}
+
+type AddPiecesJobOptions = {
+  operations: OperationSelect[]
+  batchNumber: number
+}
+
+/**
+ * Create a worker function that adds pieces to the target repair dataset.
+ */
+function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
+  return async (options: AddPiecesJobOptions) => {
+    let completedOps = 0
+    let failedOps = 0
+    let operations: OperationSelect[] = options.operations
+
+    const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
+
+    try {
+      const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
+
+      // dedupe operations by CID on the target dataset for repairs jobs
+      if (repair.repairDataSetId == null) {
+        operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
+      }
+      group.message(`Pulling ${operations.length} pieces...`)
+      // pull pieces
+      if (operations.length > 0) {
+        const pullResult = await SP.waitForPullPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: operationsToPullPieces(operations),
+          timeout: 1000 * 60 * 30,
+          onStatus: (status) => {
+            const completed = status.pieces.filter((piece) => piece.status === 'complete').length
+            const failed = status.pieces.filter((piece) => piece.status === 'failed').length
+            group.message(`Pull ${completed} completed, ${failed} failed`)
+          },
+        })
+
+        for (const { pieceCid, status } of pullResult.pieces) {
+          const cid = pieceCid.toString()
+          if (status !== 'complete') {
+            state.failedOperations++
+            failedOps++
+            const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
+            operations = operationToCommit
+            await upsertOperations({
+              localDb,
+              operations: operationToFailed.map((operation) => ({
+                ...operation,
+                status: 'failed',
+                error: `pull failed with status ${status}`,
+              })),
+            })
+          }
+        }
+      }
+
+      // add pieces
+      if (operations.length > 0) {
+        group.message(`Adding ${operations.length} pieces...`)
+        const addPiecesResult = await SP.addPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: operations.map((operation) => ({
+            pieceCid: Piece.from(operation.cid),
+            metadata: operation.metadata,
+          })),
+        })
+
+        group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
+        const addPiecesResult2 = await SP.waitForAddPieces(addPiecesResult)
+        state.completedOperations += operations.length
+        completedOps += operations.length
+        await upsertOperations({
+          localDb,
+          operations: operations.map((operation) => ({
+            ...operation,
+            status: 'completed',
+            error: null,
+            result: { dataSetId: addPiecesResult2.dataSetId, txHash: addPiecesResult2.txHash },
+          })),
+        })
+      }
+      group.message(`Done. ${completedOps} added, ${failedOps} failed`)
+    } catch (error) {
+      state.failedOperations += operations.length
+      const message = error instanceof Error ? error.message : 'Unknown error'
+      group.message(`${message.replace(/\n/g, ' ')}`)
+      await upsertOperations({
+        localDb,
+        operations: operations.map((operation) => ({
+          ...operation,
+          status: 'failed',
+          error: message,
+        })),
+      })
+    }
+  }
+}
+
+/**
+ * Add pieces to the target dataset.
+ */
+export async function runAddPieces({
+  localDb,
+  indexerDb,
+  repair,
+  concurrency,
+  batchSize,
+  client,
+}: RunPullPiecesPhaseOptions): Promise<void> {
+  const localSchema = localDb._.fullSchema
+  let cursor = 0
+
+  const totalOperations = await localDb.$count(
+    localSchema.operations,
+    and(
+      eq(localSchema.operations.repairId, repair.id),
+      eq(localSchema.operations.type, 'add_piece'),
+      inArray(localSchema.operations.status, ['pending', 'failed'])
+    )
+  )
+  let batchNumber = 0
+  const state = {
+    totalBatches: Math.ceil(totalOperations / batchSize),
+    totalOperations,
+    completedOperations: 0,
+    failedOperations: 0,
+  }
+
+  const log = taskLog({
+    title: 'Adding pieces',
+    limit: 1,
+  })
+
+  async function getNextBatch(): Promise<OperationSelect[] | null> {
+    const operations = await localDb.query.operations.findMany({
+      where: and(
+        eq(localSchema.operations.repairId, repair.id),
+        eq(localSchema.operations.type, 'add_piece'),
+        inArray(localSchema.operations.status, ['pending', 'failed']),
+        gt(localSchema.operations.id, cursor)
+      ),
+      orderBy: [asc(localSchema.operations.id)],
+      limit: batchSize,
+    })
+    if (operations.length === 0) {
+      return null
+    }
+
+    cursor = operations.at(-1)?.id ?? cursor
+    return operations
+  }
+
+  const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log })
+  const queue = new PQueue({ concurrency })
+
+  while (true) {
+    await queue.onSizeLessThan(concurrency)
+    const operations = await getNextBatch()
+    if (!operations) break
+    batchNumber++
+    const currentBatchNumber = batchNumber
+    queue.add(() => addPiecesJob({ operations, batchNumber: currentBatchNumber })).catch(console.error)
+  }
+
+  await queue.onIdle()
+
+  log.success(`Added ${state.completedOperations} pieces, ${state.failedOperations} failed`, { showLog: true })
+
+  await repairUpdate({
+    localDb,
+    repairId: repair.id,
+    status: state.failedOperations > 0 ? 'failed' : 'completed',
+  })
+}

package/src/pipeline/create-datasets.ts

@@ -1,13 +1,15 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
-import { getRepairDataset } from '../db/get-repair-dataset.ts'
+import { eq } from 'drizzle-orm'
+import { findRepairDataset } from '../db/find-repair-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import type { RepairSelect } from '../local-schema.ts'
 import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
-import { getRepairDatasetMetadata, hashLink } from '../utils.ts'
+import { hashLink } from '../utils.ts'
 
 export type EnsureRepairDatasetOptions = {
+  source: string
   localDb: LocalDatabase
   indexerDb: IndexerDatabase
   client: WalletClient
@@ -19,7 +21,7 @@ export type EnsureRepairDatasetOptions = {
  *
  * @param options - The options for ensuring the repair dataset.
  */
-export async function ensureRepairDataset({ localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
+export async function ensureRepairDataset({ source, localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring repair dataset',
   })
@@ -31,10 +33,11 @@ export async function ensureRepairDataset({ localDb, indexerDb, client, repair }
 
   let datasetId: bigint | null = null
   // check if dataset already exists
-  const existingDatasetId = await getRepairDataset({
+  const existingDatasetId = await findRepairDataset({
     indexerDb,
     providerId: repair.targetProviderId,
     payer: client.account.address,
+    source,
   })
 
   if (existingDatasetId) {
@@ -46,7 +49,10 @@ export async function ensureRepairDataset({ localDb, indexerDb, client, repair }
       serviceURL: provider.pdp.serviceURL,
       payer: client.account.address,
       cdn: false,
-      metadata: getRepairDatasetMetadata(),
+      metadata: {
+        source,
+        withIPFSIndexing: '',
+      },
     })
     log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
     const waitForResult = await SP.waitForCreateDataSet({
@@ -62,3 +68,63 @@ export async function ensureRepairDataset({ localDb, indexerDb, client, repair }
   })
   return datasetId
 }
+
+/**
+ * Ensure a replication target dataset exists by creating a fresh dataset with source metadata.
+ *
+ * @param options - The options for ensuring the replication dataset.
+ */
+export async function ensureReplicateDataset({ localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
+  const log = p.taskLog({
+    title: 'Ensuring replication dataset',
+  })
+
+  if (repair.repairDataSetId == null) {
+    throw new Error('Missing source dataset ID')
+  }
+
+  const provider = await getPDPProvider(client, {
+    providerId: repair.targetProviderId,
+  })
+
+  if (!provider) throw new Error(`Target provider ${repair.targetProviderId} not found or inactive`)
+
+  if (repair.targetDataSetId != null) {
+    log.success(`Data set #${repair.targetDataSetId} already exists at ${provider.pdp.serviceURL}`)
+    return repair.targetDataSetId
+  }
+
+  const schema = indexerDb._.fullSchema
+  const sourceDataSet = await indexerDb.query.dataSets.findFirst({
+    where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
+    columns: {
+      metadata: true,
+      withCdn: true,
+    },
+  })
+
+  if (!sourceDataSet) {
+    throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
+  }
+
+  const { txHash, statusUrl } = await SP.createDataSet(client, {
+    payee: provider.payee,
+    serviceURL: provider.pdp.serviceURL,
+    payer: client.account.address,
+    cdn: sourceDataSet.withCdn,
+    metadata: sourceDataSet.metadata ?? undefined,
+  })
+  log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
+  const waitForResult = await SP.waitForCreateDataSet({
+    statusUrl,
+  })
+  const datasetId = waitForResult.dataSetId
+  log.success(`Data set #${datasetId} created at ${provider.pdp.serviceURL}`)
+
+  await repairUpdate({
+    localDb,
+    repairId: repair.id,
+    targetDataSetId: datasetId,
+  })
+  return datasetId
+}