@filoz/repair-cli 0.4.0 → 1.1.0

package/src/db/get-repair-provider.ts

@@ -1,47 +1,27 @@
-import { and, eq } from 'drizzle-orm'
+import type { Address } from 'viem'
+import { getProvider } from '../api.ts'
 import type { Context, RepairProvider } from '../types.ts'
 
-export interface GetRepairProviderOptions extends Pick<Context, 'indexerDb'> {
+export interface GetRepairProviderOptions extends Pick<Context, 'api'> {
   providerId: bigint
 }
 
 /**
  * Load an active provider by ID for use as a repair target.
  */
-export async function getRepairProvider({
-  indexerDb,
-  providerId,
-}: GetRepairProviderOptions): Promise<RepairProvider | null> {
-  const schema = indexerDb._.fullSchema
-  const [provider] = await indexerDb
-    .select({
-      providerId: schema.providers.providerId,
-      providerAddress: schema.providers.providerAddress,
-      name: schema.providers.name,
-      serviceUrl: schema.providers.serviceUrl,
-      approved: schema.providers.approved,
-      endorsed: schema.providers.endorsed,
-    })
-    .from(schema.providers)
-    .where(
-      and(
-        eq(schema.providers.providerId, providerId),
-        eq(schema.providers.providerActive, true),
-        eq(schema.providers.pdpProductActive, true)
-      )
-    )
-    .limit(1)
+export async function getRepairProvider({ api, providerId }: GetRepairProviderOptions): Promise<RepairProvider | null> {
+  const provider = await getProvider(api, providerId)
 
-  if (!provider?.providerAddress || !provider?.serviceUrl || !provider?.name) {
+  if (!provider?.active || !provider.pdpProductActive || !provider.serviceUrl || !provider.name) {
     return null
   }
 
   return {
-    providerId: provider.providerId,
-    providerAddress: provider.providerAddress,
+    providerId: BigInt(provider.providerId),
+    providerAddress: provider.address as Address,
     name: provider.name,
     serviceUrl: provider.serviceUrl,
-    approved: provider.approved,
-    endorsed: provider.endorsed,
+    approved: provider.approved ?? false,
+    endorsed: provider.endorsed ?? false,
   }
 }

package/src/db/repair-create.ts

@@ -18,7 +18,7 @@ export interface RepairCreateOptions extends Context {
  * @returns {Promise<number>} The ID of the created repair.
  */
 export async function repairCreate(options: RepairCreateOptions): Promise<number> {
-  const { indexerDb, localDb, repairProviderId, targetProviderId, client } = options
+  const { api, localDb, repairProviderId, targetProviderId, client } = options
   const localSchema = localDb._.fullSchema
   const now = Date.now()
   const blockNumber = await getBlockNumber(client)
@@ -34,7 +34,7 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
     throw new RepairCreationError('Target provider must differ from the provider being repaired')
   }
   const targetProvider = await getRepairProvider({
-    indexerDb,
+    api,
     providerId: targetProviderId,
   })
 
@@ -68,7 +68,7 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
   const seenCids = new Set<string>()
   await forEachPiecesPage(
     {
-      indexerDb,
+      api,
       providerId: repairProviderId,
       repairId: repair.id,
       blockNumber,

package/src/db/replicate-create.ts

@@ -1,6 +1,6 @@
 import { taskLog } from '@clack/prompts'
-import { eq } from 'drizzle-orm'
 import { getBlockNumber } from 'viem/actions'
+import { getDataSet } from '../api.ts'
 import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
 import type { Context } from '../types.ts'
 import { forEachDataSetPiecesPage } from './get-pieces.ts'
@@ -18,8 +18,7 @@ export interface ReplicateCreateOptions extends Context {
  * @returns The ID of the created local job.
  */
 export async function replicateCreate(options: ReplicateCreateOptions): Promise<number> {
-  const { indexerDb, localDb, dataSetId, targetProviderId, client } = options
-  const indexerSchema = indexerDb._.fullSchema
+  const { api, localDb, dataSetId, targetProviderId, client } = options
   const localSchema = localDb._.fullSchema
   const now = Date.now()
   const blockNumber = await getBlockNumber(client)
@@ -30,35 +29,28 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
     retainLog: true,
   })
 
-  const [sourceDataSet] = await indexerDb
-    .select({
-      dataSetId: indexerSchema.dataSets.dataSetId,
-      providerId: indexerSchema.dataSets.providerId,
-      deleted: indexerSchema.dataSets.deleted,
-    })
-    .from(indexerSchema.dataSets)
-    .where(eq(indexerSchema.dataSets.dataSetId, dataSetId))
-    .limit(1)
+  const sourceDataSet = await getDataSet(api, dataSetId)
 
-  if (!sourceDataSet || sourceDataSet.deleted) {
+  if (!sourceDataSet || sourceDataSet.deleted || sourceDataSet.providerId == null) {
     throw new RepairCreationError(`Source dataset ${dataSetId} not found or deleted`)
   }
+  const sourceProviderId = BigInt(sourceDataSet.providerId)
 
-  if (targetProviderId === sourceDataSet.providerId) {
+  if (targetProviderId === sourceProviderId) {
     throw new RepairCreationError('Target provider must differ from the source dataset provider')
   }
 
   const sourceProvider = await getRepairProvider({
-    indexerDb,
-    providerId: sourceDataSet.providerId,
+    api,
+    providerId: sourceProviderId,
   })
 
   if (!sourceProvider) {
-    throw new RepairCreationError(`Source provider ${sourceDataSet.providerId} not found or inactive`)
+    throw new RepairCreationError(`Source provider ${sourceProviderId} not found or inactive`)
   }
 
   const targetProvider = await getRepairProvider({
-    indexerDb,
+    api,
     providerId: targetProviderId,
   })
 
@@ -71,7 +63,7 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
     .values({
       kind: 'replicate',
       repairProviderId: sourceProvider.providerId,
-      repairDataSetId: sourceDataSet.dataSetId,
+      repairDataSetId: dataSetId,
       sourceProviderUrl: sourceProvider.serviceUrl,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
@@ -87,8 +79,8 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
   let totalOperations = 0
   await forEachDataSetPiecesPage(
     {
-      indexerDb,
-      dataSetId: sourceDataSet.dataSetId,
+      api,
+      dataSetId,
       repairId: repair.id,
       sourceProviderUrl: sourceProvider.serviceUrl,
     },

package/src/db/second-copy-create.ts

@@ -76,7 +76,7 @@ export async function parsePieceCidsFromFile(piecesFile: string): Promise<string
  * @returns The ID of the created local job.
  */
 export async function secondCopyCreate(options: SecondCopyCreateOptions): Promise<number> {
-  const { indexerDb, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
+  const { api, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
   const localSchema = localDb._.fullSchema
   const now = Date.now()
   const blockNumber = await getBlockNumber(client)
@@ -89,7 +89,7 @@ export async function secondCopyCreate(options: SecondCopyCreateOptions): Promis
   })
 
   const targetProvider = await getRepairProvider({
-    indexerDb,
+    api,
     providerId: targetProviderId,
   })
 

package/src/middleware.ts

@@ -1,25 +1,24 @@
-import type { Chain } from '@filoz/synapse-core/chains'
-import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres'
+import type { FilecoinChain } from '@filoz/synapse-core/chains'
 import { middleware, z } from 'incur'
 import type { Account, Client, Transport } from 'viem'
-import * as indexerSchema from './indexer-schema.ts'
-import type { IndexerDatabase, LocalDatabase } from './types.ts'
+import { createIndexerApi } from './api.ts'
+import type { IndexerApi, LocalDatabase } from './types.ts'
 import { config, createLocalDatabase, getClient } from './utils.ts'
 
 export const contextSchema = z.object({
-  indexerDb: z.custom<IndexerDatabase>(),
+  api: z.custom<IndexerApi>(),
   localDb: z.custom<LocalDatabase>(),
   config: z.custom<typeof config>(),
-  client: z.custom<Client<Transport, Chain, Account>>(),
-  chain: z.custom<Chain>(),
+  client: z.custom<Client<Transport, FilecoinChain, Account>>(),
+  chain: z.custom<FilecoinChain>(),
   source: z.string(),
   isInteractive: z.boolean(),
 })
 
 export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
-  const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl, source } = config.store
+  const { dbPath, chainId, apiUrl, source } = config.store
 
-  if (!dbPath || !chainId || !indexerMainnetUrl || !indexerCalibrationUrl || !source) {
+  if (!dbPath || !chainId || !source) {
     return c.error({
       code: 'CONFIG_NOT_SET',
       message: 'Config not set. Please run `repair setup` first.',
@@ -28,13 +27,11 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
   }
 
   const localDb = await createLocalDatabase(dbPath)
-  const indexerDb = drizzlePostgres(chainId === 314 ? indexerMainnetUrl : indexerCalibrationUrl, {
-    schema: indexerSchema,
-  })
+  const api = createIndexerApi({ baseUrl: apiUrl, chainId })
 
   const { client, chain } = getClient(chainId)
   c.set('localDb', localDb)
-  c.set('indexerDb', indexerDb)
+  c.set('api', api)
   c.set('config', config)
   c.set('client', client)
   c.set('chain', chain)
@@ -43,5 +40,4 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
   await next()
 
   localDb.$client.close()
-  await indexerDb.$client.end()
 })

package/src/pipeline/add-pieces.ts

@@ -1,32 +1,43 @@
 import { taskLog } from '@clack/prompts'
-import * as Piece from '@filoz/synapse-core/piece'
 import * as SP from '@filoz/synapse-core/sp'
 import { and, asc, eq, gt, inArray } from 'drizzle-orm'
 import PQueue from 'p-queue'
+import { listDataSetPieces } from '../api.ts'
 import { dedupeCids } from '../db/dedupe-cids.ts'
 import { getTargetDataset } from '../db/get-target-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import { upsertOperations } from '../db/upsert-operations.ts'
 import type { OperationSelect, RepairSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
-import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
+import {
+  completeConfirmedOperations,
+  excludeOperationsByCid,
+  fitAddPiecesBatch,
+  hashLink,
+  operationsToAddPieces,
+  operationsToPullPieces,
+} from '../utils.ts'
+
+/** Operations read per batch before fitting; above the ~400 metadata-free pieces one addPieces message holds. */
+const MAX_BATCH_CANDIDATES = 500
 
 export type RunPullPiecesPhaseOptions = {
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   repair: RepairSelect
   concurrency: number
-  batchSize: number
+  /** Max pieces per batch. Defaults to as many as fit in one addPieces message. */
+  batchSize?: number
   client: WalletClient
 }
 
 type CreateAddPiecesWorkerOptions = {
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  /** CIDs already in the target dataset; repairs only. Updated as batches are added. */
+  targetCids?: Set<string>
   repair: RepairSelect
   client: WalletClient
   state: {
-    totalBatches: number
     totalOperations: number
     completedOperations: number
     failedOperations: number
@@ -42,21 +53,20 @@ type AddPiecesJobOptions = {
 /**
  * Create a worker function that adds pieces to the target repair dataset.
  */
-function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
+function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
   return async (options: AddPiecesJobOptions) => {
     let completedOps = 0
     let failedOps = 0
     let operations: OperationSelect[] = options.operations
-    const isRepair = repair.repairDataSetId == null
 
-    const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
+    const group = log.group(`Batch ${options.batchNumber} (${options.operations.length} pieces)`)
 
     try {
       const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
 
       // dedupe operations by CID on the target dataset for repairs jobs
-      if (isRepair) {
-        operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
+      if (targetCids) {
+        operations = await dedupeCids({ localDb, existingCids: targetCids, operations })
       }
       // A previous run may have submitted a transaction and crashed before marking rows completed.
       const operationsBeforeConfirmationCheck = operations.length
@@ -75,7 +85,6 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
           dataSetId: dataset.dataSetId,
           clientDataSetId: dataset.clientDataSetId,
           pieces: operationsToPullPieces(operations),
-          timeout: 1000 * 60 * 30, // 30 minutes
           onStatus: (status) => {
             const completed = status.pieces.filter((piece) => piece.status === 'complete').length
             const failed = status.pieces.filter((piece) => piece.status === 'failed').length
@@ -109,10 +118,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
           serviceURL: repair.targetProviderUrl,
           dataSetId: dataset.dataSetId,
           clientDataSetId: dataset.clientDataSetId,
-          pieces: operations.map((operation) => ({
-            pieceCid: Piece.from(operation.cid),
-            metadata: isRepair ? undefined : operation.metadata,
-          })),
+          pieces: operationsToAddPieces(operations, !targetCids),
         })
         await upsertOperations({
           localDb,
@@ -124,15 +130,18 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
 
         group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
         const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
+        // Curio may replace the message by fee; record the hash that landed on chain.
+        const confirmedTxHash = waitForAddPieces.confirmedTxHash ?? waitForAddPieces.txHash
         state.completedOperations += operations.length
         completedOps += operations.length
+        for (const operation of operations) targetCids?.add(operation.cid)
         await upsertOperations({
           localDb,
           operations: operations.map((operation) => ({
             ...operation,
             status: 'completed',
             error: null,
-            txHash: waitForAddPieces.txHash,
+            txHash: confirmedTxHash,
           })),
         })
       }
@@ -158,7 +167,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
  */
 export async function runAddPieces({
   localDb,
-  indexerDb,
+  api,
   repair,
   concurrency,
   batchSize,
@@ -177,7 +186,6 @@ export async function runAddPieces({
   )
   let batchNumber = 0
   const state = {
-    totalBatches: Math.ceil(totalOperations / batchSize),
     totalOperations,
     completedOperations: 0,
     failedOperations: 0,
@@ -188,8 +196,20 @@ export async function runAddPieces({
     limit: 1,
   })
 
+  const dataset = totalOperations > 0 ? await getTargetDataset({ localDb, repairId: repair.id, client }) : undefined
+
+  // Repairs and second copies dedupe into one shared dataset; read its CIDs once instead of per batch.
+  let targetCids: Set<string> | undefined
+  if (repair.repairDataSetId == null && dataset) {
+    const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false })
+    targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])))
+  }
+
   async function getNextBatch(): Promise<OperationSelect[] | null> {
-    const operations = await localDb.query.operations.findMany({
+    if (!dataset) {
+      return null
+    }
+    const candidates = await localDb.query.operations.findMany({
       where: and(
         eq(localSchema.operations.repairId, repair.id),
         eq(localSchema.operations.type, 'add_piece'),
@@ -197,17 +217,25 @@ export async function runAddPieces({
         gt(localSchema.operations.id, cursor)
       ),
       orderBy: [asc(localSchema.operations.id)],
-      limit: batchSize,
+      limit: batchSize ?? MAX_BATCH_CANDIDATES,
     })
-    if (operations.length === 0) {
+    if (candidates.length === 0) {
       return null
     }
 
+    // Size the batch by what fits in one addPieces message rather than a fixed piece count.
+    const operations = fitAddPiecesBatch({
+      operations: candidates,
+      dataSetId: dataset.dataSetId,
+      chain: client.chain,
+      withMetadata: !targetCids,
+    })
+
     cursor = operations.at(-1)?.id ?? cursor
     return operations
   }
 
-  const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log })
+  const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log })
   const queue = new PQueue({ concurrency })
 
   while (true) {

package/src/pipeline/create-datasets.ts

@@ -1,18 +1,18 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
-import { eq } from 'drizzle-orm'
 import type { Address } from 'viem'
+import { getDataSet, toStringMetadata } from '../api.ts'
 import { findRepairDataset } from '../db/find-repair-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import type { RepairSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
 import { hashLink } from '../utils.ts'
 
 export type EnsureRepairDatasetOptions = {
   source: string
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   client: WalletClient
   repair: RepairSelect
   payer: Address
@@ -23,14 +23,7 @@ export type EnsureRepairDatasetOptions = {
  *
  * @param options - The options for ensuring the repair dataset.
  */
-export async function ensureRepairDataset({
-  source,
-  localDb,
-  indexerDb,
-  client,
-  repair,
-  payer,
-}: EnsureRepairDatasetOptions) {
+export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring repair dataset',
   })
@@ -43,7 +36,7 @@ export async function ensureRepairDataset({
   let datasetId: bigint | null = null
   // check if dataset already exists
   const existingDatasetId = await findRepairDataset({
-    indexerDb,
+    api,
     providerId: repair.targetProviderId,
     payer,
     source,
@@ -83,13 +76,7 @@ export async function ensureRepairDataset({
  *
  * @param options - The options for ensuring the replication dataset.
  */
-export async function ensureReplicateDataset({
-  localDb,
-  indexerDb,
-  client,
-  repair,
-  payer,
-}: EnsureRepairDatasetOptions) {
+export async function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring replication dataset',
   })
@@ -109,14 +96,7 @@ export async function ensureReplicateDataset({
     return repair.targetDataSetId
   }
 
-  const schema = indexerDb._.fullSchema
-  const sourceDataSet = await indexerDb.query.dataSets.findFirst({
-    where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
-    columns: {
-      metadata: true,
-      withCdn: true,
-    },
-  })
+  const sourceDataSet = await getDataSet(api, repair.repairDataSetId)
 
   if (!sourceDataSet) {
     throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
@@ -126,8 +106,8 @@ export async function ensureReplicateDataset({
     payee: provider.payee,
     serviceURL: provider.pdp.serviceURL,
     payer,
-    cdn: sourceDataSet.withCdn,
-    metadata: sourceDataSet.metadata ?? undefined,
+    cdn: sourceDataSet.withCdn ?? false,
+    metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
   })
   log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
   const waitForResult = await SP.waitForCreateDataSet({

package/src/types.ts

@@ -1,30 +1,32 @@
-import type { Chain } from '@filoz/synapse-core/chains'
+import type { ApiClient } from '@filecoin-project/api-client'
+import type { FilecoinChain } from '@filoz/synapse-core/chains'
 import type { Client } from '@libsql/client'
 import type { LibSQLDatabase } from 'drizzle-orm/libsql'
-import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
 import type { z } from 'incur'
-import type { Pool } from 'pg'
 import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem'
-import type * as indexerSchema from './indexer-schema.ts'
+import type { Network } from './api.ts'
 import type * as localSchema from './local-schema.ts'
 import type { contextSchema } from './middleware.ts'
 export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
   $client: Client
 }
 
-export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
-  $client: Pool
+/**
+ * filecoin-api client bound to the configured network.
+ */
+export type IndexerApi = {
+  client: ApiClient
+  network: Network
 }
 
 export interface Config {
   privateKey: Hex
-  indexerMainnetUrl: string
-  indexerCalibrationUrl: string
+  apiUrl: string
   chainId: number
   dbPath: string
 }
 
-export type WalletClient = ViemClient<Transport, Chain, Account>
+export type WalletClient = ViemClient<Transport, FilecoinChain, Account>
 
 export type Context = z.infer<typeof contextSchema>
 

package/src/utils.ts

@@ -1,9 +1,8 @@
-import { type Chain, getChain } from '@filoz/synapse-core/chains'
+import { type FilecoinChain, getChain } from '@filoz/synapse-core/chains'
 import * as Piece from '@filoz/synapse-core/piece'
-import type * as SP from '@filoz/synapse-core/sp'
+import * as SP from '@filoz/synapse-core/sp'
 import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
-import type { PgTable } from 'drizzle-orm/pg-core'
 import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
 import { z } from 'incur'
 import { Conf } from 'iso-conf'
@@ -20,8 +19,7 @@ import type { LocalDatabase, WalletClient } from './types.ts'
 
 export const configSchema = z.object({
   privateKey: z.string().optional(),
-  indexerMainnetUrl: z.url().optional(),
-  indexerCalibrationUrl: z.url().optional(),
+  apiUrl: z.url().optional(),
   chainId: z.number().optional(),
   dbPath: z.string().optional(),
   source: z.string().optional(),
@@ -168,7 +166,7 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
  * @param chain - The chain to use
  * @returns The link
  */
-export function hashLink(hash: string, chain: Chain) {
+export function hashLink(hash: string, chain: FilecoinChain) {
   const link = terminalLink(hash, `${chain.blockExplorers?.default?.url}/tx/${hash}`)
   return link
 }
@@ -273,7 +271,7 @@ export async function findPieceOnProviders(providers: string[], pieceCid: string
   return result
 }
 
-export const buildConflictUpdateColumns = <T extends PgTable | SQLiteTable, Q extends keyof T['_']['columns']>(
+export const buildConflictUpdateColumns = <T extends SQLiteTable, Q extends keyof T['_']['columns']>(
   table: T,
   columns?: Q[]
 ) => {
@@ -329,3 +327,69 @@ export function operationsToPullPieces(operations: OperationSelect[]) {
   }
   return pieces
 }
+
+/**
+ * Convert operations to addPieces inputs. Repairs and second copies add pieces without metadata.
+ */
+export function operationsToAddPieces(operations: OperationSelect[], withMetadata: boolean) {
+  return operations.map((operation) => ({
+    pieceCid: Piece.from(operation.cid),
+    metadata: withMetadata ? operation.metadata : undefined,
+  }))
+}
+
+/**
+ * Take the longest prefix of operations that fits in one addPieces message.
+ * Always returns at least one operation so an oversized or invalid piece fails in the worker instead of stalling the queue.
+ */
+export function fitAddPiecesBatch({
+  operations,
+  dataSetId,
+  chain,
+  withMetadata,
+}: {
+  operations: OperationSelect[]
+  dataSetId: bigint
+  chain: FilecoinChain
+  withMetadata: boolean
+}): OperationSelect[] {
+  // Malformed CIDs throw; stop before the first one so it fails alone in the worker.
+  const pieces: SP.LimiterPiece[] = []
+  for (const operation of operations) {
+    try {
+      pieces.push(...operationsToAddPieces([operation], withMetadata))
+    } catch {
+      break
+    }
+  }
+  if (pieces.length === 0) {
+    return operations.slice(0, 1)
+  }
+
+  const fits = (count: number) => {
+    try {
+      return SP.addPiecesFits({
+        kind: 'addPieces',
+        dataSetId,
+        legacyPieceStorageIdLimit: chain.legacyPieceStorageIdLimit,
+        pieces: pieces.slice(0, count),
+      })
+    } catch {
+      // Invalid metadata throws; isolate that piece so addPieces fails it in the worker.
+      return false
+    }
+  }
+
+  // Calldata only grows as pieces are added, so binary search the largest prefix that fits.
+  let low = 1
+  let high = pieces.length
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2)
+    if (fits(mid)) {
+      low = mid
+    } else {
+      high = mid - 1
+    }
+  }
+  return operations.slice(0, low)
+}

package/dist/src/db/find-providers-by-cid.d.ts

@@ -1,9 +0,0 @@
-import type { IndexerDatabase, RepairProvider } from '../types.ts';
-export type FindProvidersByCidOptions = {
-    indexerDb: IndexerDatabase;
-    cids: readonly string[];
-    blockNumber: bigint;
-};
-export type ProvidersByCid = Record<string, RepairProvider[]>;
-export declare function findProvidersByCid({ indexerDb, cids, blockNumber, }: FindProvidersByCidOptions): Promise<ProvidersByCid>;
-//# sourceMappingURL=find-providers-by-cid.d.ts.map

package/dist/src/db/find-providers-by-cid.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"find-providers-by-cid.d.ts","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,eAAe,CAAA;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAKD,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAA;AAS7D,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACZ,EAAE,yBAAyB,GAAG,OAAO,CAAC,cAAc,CAAC,CA4CrD"}