@filoz/repair-cli 1.0.0 → 1.1.1

package/src/pipeline/add-pieces.ts

@@ -1,5 +1,4 @@
 import { taskLog } from '@clack/prompts'
-import * as Piece from '@filoz/synapse-core/piece'
 import * as SP from '@filoz/synapse-core/sp'
 import { and, asc, eq, gt, inArray } from 'drizzle-orm'
 import PQueue from 'p-queue'
@@ -10,14 +9,35 @@ import { repairUpdate } from '../db/repair-update.ts'
 import { upsertOperations } from '../db/upsert-operations.ts'
 import type { OperationSelect, RepairSelect } from '../local-schema.ts'
 import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
-import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
+import {
+  completeConfirmedOperations,
+  excludeOperationsByCid,
+  fitAddPiecesBatch,
+  hashLink,
+  operationsToAddPieces,
+  operationsToPullPieces,
+} from '../utils.ts'
+
+/** Operations read per batch before fitting; above the ~400 metadata-free pieces one addPieces message holds. */
+const MAX_BATCH_CANDIDATES = 500
+
+/**
+ * Curio admits at most 108 unique pieces per pull request and 108 pending pulled pieces per client.
+ *
+ * @see https://github.com/filecoin-project/curio/blob/main/pdp/pull_types.go
+ */
+const MAX_PULL_PIECES = 108
+
+/** Curio answers 429 with Retry-After (1-5 minutes) while its pull queue is full; wait it out instead of failing. */
+const PULL_RETRY_COUNT = 10
 
 export type RunPullPiecesPhaseOptions = {
   localDb: LocalDatabase
   api: IndexerApi
   repair: RepairSelect
   concurrency: number
-  batchSize: number
+  /** Max pieces per batch. Defaults to as many as fit in one addPieces message. */
+  batchSize?: number
   client: WalletClient
 }
 
@@ -25,10 +45,11 @@ type CreateAddPiecesWorkerOptions = {
   localDb: LocalDatabase
   /** CIDs already in the target dataset; repairs only. Updated as batches are added. */
   targetCids?: Set<string>
+  /** Max unique pieces per pull request. */
+  pullChunkSize: number
   repair: RepairSelect
   client: WalletClient
   state: {
-    totalBatches: number
     totalOperations: number
     completedOperations: number
     failedOperations: number
@@ -44,13 +65,21 @@ type AddPiecesJobOptions = {
 /**
  * Create a worker function that adds pieces to the target repair dataset.
  */
-function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
+function createAddPiecesWorker({
+  localDb,
+  targetCids,
+  pullChunkSize,
+  repair,
+  client,
+  state,
+  log,
+}: CreateAddPiecesWorkerOptions) {
   return async (options: AddPiecesJobOptions) => {
     let completedOps = 0
     let failedOps = 0
     let operations: OperationSelect[] = options.operations
 
-    const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
+    const group = log.group(`Batch ${options.batchNumber} (${options.operations.length} pieces)`)
 
     try {
       const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
@@ -69,36 +98,41 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
       }
 
       group.message(`Pulling ${operations.length} pieces...`)
-      // pull pieces
+      // pull pieces in chunks that Curio admits, one chunk at a time per worker
       if (operations.length > 0) {
-        const pullResult = await SP.waitForPullPieces(client, {
-          serviceURL: repair.targetProviderUrl,
-          dataSetId: dataset.dataSetId,
-          clientDataSetId: dataset.clientDataSetId,
-          pieces: operationsToPullPieces(operations),
-          timeout: 1000 * 60 * 30, // 30 minutes
-          onStatus: (status) => {
-            const completed = status.pieces.filter((piece) => piece.status === 'complete').length
-            const failed = status.pieces.filter((piece) => piece.status === 'failed').length
-            group.message(`Pull ${completed} completed, ${failed} failed`)
-          },
-        })
+        const pullPieces = operationsToPullPieces(operations)
+        const pullChunks = Math.ceil(pullPieces.length / pullChunkSize)
+        for (let offset = 0; offset < pullPieces.length; offset += pullChunkSize) {
+          const chunk = offset / pullChunkSize + 1
+          const pullResult = await SP.waitForPullPieces(client, {
+            serviceURL: repair.targetProviderUrl,
+            dataSetId: dataset.dataSetId,
+            clientDataSetId: dataset.clientDataSetId,
+            pieces: pullPieces.slice(offset, offset + pullChunkSize),
+            retryCount: PULL_RETRY_COUNT,
+            onStatus: (status) => {
+              const completed = status.pieces.filter((piece) => piece.status === 'complete').length
+              const failed = status.pieces.filter((piece) => piece.status === 'failed').length
+              group.message(`Pull ${chunk}/${pullChunks}: ${completed} completed, ${failed} failed`)
+            },
+          })
 
-        for (const { pieceCid, status } of pullResult.pieces) {
-          const cid = pieceCid.toString()
-          if (status !== 'complete') {
-            state.failedOperations++
-            failedOps++
-            const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
-            operations = operationToCommit
-            await upsertOperations({
-              localDb,
-              operations: operationToFailed.map((operation) => ({
-                ...operation,
-                status: 'failed',
-                error: `pull failed with status ${status}`,
-              })),
-            })
+          for (const { pieceCid, status } of pullResult.pieces) {
+            const cid = pieceCid.toString()
+            if (status !== 'complete') {
+              state.failedOperations++
+              failedOps++
+              const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
+              operations = operationToCommit
+              await upsertOperations({
+                localDb,
+                operations: operationToFailed.map((operation) => ({
+                  ...operation,
+                  status: 'failed',
+                  error: `pull failed with status ${status}`,
+                })),
+              })
+            }
           }
         }
       }
@@ -110,10 +144,7 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
           serviceURL: repair.targetProviderUrl,
           dataSetId: dataset.dataSetId,
           clientDataSetId: dataset.clientDataSetId,
-          pieces: operations.map((operation) => ({
-            pieceCid: Piece.from(operation.cid),
-            metadata: targetCids ? undefined : operation.metadata,
-          })),
+          pieces: operationsToAddPieces(operations, !targetCids),
         })
         await upsertOperations({
           localDb,
@@ -125,6 +156,8 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
 
         group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
         const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
+        // Curio may replace the message by fee; record the hash that landed on chain.
+        const confirmedTxHash = waitForAddPieces.confirmedTxHash ?? waitForAddPieces.txHash
         state.completedOperations += operations.length
         completedOps += operations.length
         for (const operation of operations) targetCids?.add(operation.cid)
@@ -134,7 +167,7 @@ function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log
             ...operation,
             status: 'completed',
             error: null,
-            txHash: waitForAddPieces.txHash,
+            txHash: confirmedTxHash,
           })),
         })
       }
@@ -179,7 +212,6 @@ export async function runAddPieces({
   )
   let batchNumber = 0
   const state = {
-    totalBatches: Math.ceil(totalOperations / batchSize),
     totalOperations,
     completedOperations: 0,
     failedOperations: 0,
@@ -190,16 +222,20 @@ export async function runAddPieces({
     limit: 1,
   })
 
+  const dataset = totalOperations > 0 ? await getTargetDataset({ localDb, repairId: repair.id, client }) : undefined
+
   // Repairs and second copies dedupe into one shared dataset; read its CIDs once instead of per batch.
   let targetCids: Set<string> | undefined
-  if (repair.repairDataSetId == null && totalOperations > 0) {
-    const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
+  if (repair.repairDataSetId == null && dataset) {
     const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false })
     targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])))
   }
 
   async function getNextBatch(): Promise<OperationSelect[] | null> {
-    const operations = await localDb.query.operations.findMany({
+    if (!dataset) {
+      return null
+    }
+    const candidates = await localDb.query.operations.findMany({
       where: and(
         eq(localSchema.operations.repairId, repair.id),
         eq(localSchema.operations.type, 'add_piece'),
@@ -207,17 +243,27 @@ export async function runAddPieces({
         gt(localSchema.operations.id, cursor)
       ),
       orderBy: [asc(localSchema.operations.id)],
-      limit: batchSize,
+      limit: batchSize ?? MAX_BATCH_CANDIDATES,
     })
-    if (operations.length === 0) {
+    if (candidates.length === 0) {
       return null
     }
 
+    // Size the batch by what fits in one addPieces message rather than a fixed piece count.
+    const operations = fitAddPiecesBatch({
+      operations: candidates,
+      dataSetId: dataset.dataSetId,
+      chain: client.chain,
+      withMetadata: !targetCids,
+    })
+
     cursor = operations.at(-1)?.id ?? cursor
     return operations
   }
 
-  const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log })
+  // Keep pieces pulling across all workers within Curio's per-client limit.
+  const pullChunkSize = Math.max(1, Math.floor(MAX_PULL_PIECES / concurrency))
+  const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, pullChunkSize, repair, client, state, log })
   const queue = new PQueue({ concurrency })
 
   while (true) {

package/src/types.ts

@@ -1,5 +1,5 @@
 import type { ApiClient } from '@filecoin-project/api-client'
-import type { Chain } from '@filoz/synapse-core/chains'
+import type { FilecoinChain } from '@filoz/synapse-core/chains'
 import type { Client } from '@libsql/client'
 import type { LibSQLDatabase } from 'drizzle-orm/libsql'
 import type { z } from 'incur'
@@ -26,7 +26,7 @@ export interface Config {
   dbPath: string
 }
 
-export type WalletClient = ViemClient<Transport, Chain, Account>
+export type WalletClient = ViemClient<Transport, FilecoinChain, Account>
 
 export type Context = z.infer<typeof contextSchema>
 

package/src/utils.ts

@@ -1,6 +1,6 @@
-import { type Chain, getChain } from '@filoz/synapse-core/chains'
+import { type FilecoinChain, getChain } from '@filoz/synapse-core/chains'
 import * as Piece from '@filoz/synapse-core/piece'
-import type * as SP from '@filoz/synapse-core/sp'
+import * as SP from '@filoz/synapse-core/sp'
 import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
 import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
@@ -166,7 +166,7 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
  * @param chain - The chain to use
  * @returns The link
  */
-export function hashLink(hash: string, chain: Chain) {
+export function hashLink(hash: string, chain: FilecoinChain) {
   const link = terminalLink(hash, `${chain.blockExplorers?.default?.url}/tx/${hash}`)
   return link
 }
@@ -327,3 +327,69 @@ export function operationsToPullPieces(operations: OperationSelect[]) {
   }
   return pieces
 }
+
+/**
+ * Convert operations to addPieces inputs. Repairs and second copies add pieces without metadata.
+ */
+export function operationsToAddPieces(operations: OperationSelect[], withMetadata: boolean) {
+  return operations.map((operation) => ({
+    pieceCid: Piece.from(operation.cid),
+    metadata: withMetadata ? operation.metadata : undefined,
+  }))
+}
+
+/**
+ * Take the longest prefix of operations that fits in one addPieces message.
+ * Always returns at least one operation so an oversized or invalid piece fails in the worker instead of stalling the queue.
+ */
+export function fitAddPiecesBatch({
+  operations,
+  dataSetId,
+  chain,
+  withMetadata,
+}: {
+  operations: OperationSelect[]
+  dataSetId: bigint
+  chain: FilecoinChain
+  withMetadata: boolean
+}): OperationSelect[] {
+  // Malformed CIDs throw; stop before the first one so it fails alone in the worker.
+  const pieces: SP.LimiterPiece[] = []
+  for (const operation of operations) {
+    try {
+      pieces.push(...operationsToAddPieces([operation], withMetadata))
+    } catch {
+      break
+    }
+  }
+  if (pieces.length === 0) {
+    return operations.slice(0, 1)
+  }
+
+  const fits = (count: number) => {
+    try {
+      return SP.addPiecesFits({
+        kind: 'addPieces',
+        dataSetId,
+        legacyPieceStorageIdLimit: chain.legacyPieceStorageIdLimit,
+        pieces: pieces.slice(0, count),
+      })
+    } catch {
+      // Invalid metadata throws; isolate that piece so addPieces fails it in the worker.
+      return false
+    }
+  }
+
+  // Calldata only grows as pieces are added, so binary search the largest prefix that fits.
+  let low = 1
+  let high = pieces.length
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2)
+    if (fits(mid)) {
+      low = mid
+    } else {
+      high = mid - 1
+    }
+  }
+  return operations.slice(0, low)
+}