@filoz/repair-cli 0.3.0 → 1.0.0

package/src/db/get-alternate-provider-urls.ts

@@ -0,0 +1,43 @@
+import { listProviders, type Provider } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
+
+/** Rank endorsed providers first, then approved, then by ascending ID. */
+function compareProviders(a: Provider, b: Provider): number {
+  if (a.endorsed !== b.endorsed) return a.endorsed ? -1 : 1
+  if (a.approved !== b.approved) return a.approved ? -1 : 1
+  const x = BigInt(a.providerId)
+  const y = BigInt(b.providerId)
+  return x < y ? -1 : x > y ? 1 : 0
+}
+
+/**
+ * Service URLs of the providers to probe for a copy of a repaired piece: approved or endorsed
+ * providers with a service URL, then the provider being repaired as a last resort, so pieces only
+ * it stores can still be pulled while it serves them.
+ *
+ * Callers probe these with `findPieceOnProviders`, which keeps this order, so endorsed providers
+ * are preferred as pull sources and the repaired provider is used only when no other one serves
+ * the piece.
+ */
+export async function getAlternateProviderUrls({
+  api,
+  repairProviderId,
+}: {
+  api: IndexerApi
+  repairProviderId: bigint
+}): Promise<string[]> {
+  const providers = await listProviders(api)
+  const repairProviderUrl = providers.find((provider) => BigInt(provider.providerId) === repairProviderId)?.serviceUrl
+  const urls = providers
+    .filter(
+      (provider) =>
+        (provider.approved || provider.endorsed) &&
+        provider.serviceUrl &&
+        BigInt(provider.providerId) !== repairProviderId
+    )
+    .sort(compareProviders)
+    .map((provider) => provider.serviceUrl as string)
+
+  if (repairProviderUrl) urls.push(repairProviderUrl)
+  return urls
+}

package/src/db/get-pieces.ts

@@ -1,157 +1,70 @@
 import * as Piece from '@filoz/synapse-core/piece'
-import { and, asc, eq, isNull, lte, or } from 'drizzle-orm'
 import pMap from 'p-map'
+import { listDataSetPieces, MAX_PAGE_SIZE, providerPiecePages, toStringMetadata } from '../api.ts'
 import type { OperationInsert } from '../local-schema.ts'
-import type { IndexerDatabase } from '../types.ts'
-import { findProvidersByCid } from './find-providers-by-cid.ts'
+import type { IndexerApi } from '../types.ts'
+import { getActiveDataSetIds } from './get-active-data-set-ids.ts'
+import { getAlternateProviderUrls } from './get-alternate-provider-urls.ts'
 
-/** Default page size when paginating pieces from the indexer. */
-export const DEFAULT_PIECES_PAGE_SIZE = 3000
-
-/** Options for fetching one page of `add_piece` operations for a repair. */
-export type GetPiecesPageOptions = {
-  indexerDb: IndexerDatabase
+/** Options for walking `add_piece` operations for a repair. */
+export type ForEachPiecesPageOptions = {
+  api: IndexerApi
   /** Source provider whose pieces are being repaired. */
   providerId: bigint
   /** Local repair row to attach operations to. */
   repairId: number
   /** Chain block number captured when the repair was created. */
   blockNumber: bigint
-  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
-  limit?: number
-  /** SQL offset for the indexer query. */
-  offset?: number
-  /**
-   * CIDs already emitted across prior pages. Pass the value returned from the previous call
-   * so the same CID is not queued twice when it appears in multiple source datasets.
-   */
-  seenCids?: Set<string>
-}
-
-/** Result of a single {@link getPiecesPage} call. */
-export type GetPiecesPageResult = {
-  /** `add_piece` operations ready to insert for this page (may include `skipped` rows). */
-  operations: OperationInsert[]
-  /** Whether another indexer page may exist after this one. */
-  hasMore: boolean
-  /** Offset to pass as `offset` on the next page. */
-  nextOffset: number
-  /** Updated dedupe set; pass into the next {@link getPiecesPage} call. */
-  seenCids: Set<string>
 }
 
-/** Options for {@link forEachPiecesPage}; pagination state is managed internally. */
-export type ForEachPiecesPageOptions = Omit<GetPiecesPageOptions, 'offset' | 'seenCids'>
-
-/** Options for fetching one page of source-dataset pieces for replication. */
-export type GetDataSetPiecesPageOptions = {
-  indexerDb: IndexerDatabase
+/** Options for walking source-dataset pieces for replication. */
+export type ForEachDataSetPiecesPageOptions = {
+  api: IndexerApi
   /** Source dataset whose pieces are being replicated. */
   dataSetId: bigint
   /** Local job row to attach operations to. */
   repairId: number
   /** Provider URL that serves the source dataset pieces. */
   sourceProviderUrl: string
-  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
-  limit?: number
-  /** SQL offset for the indexer query. */
-  offset?: number
 }
 
-/** Result of a single {@link getDataSetPiecesPage} call. */
-export type GetDataSetPiecesPageResult = {
-  /** `add_piece` operations ready to insert for this page. */
+/** One page of operations ready to insert into the local database. */
+export type PiecesPage = {
+  /** `add_piece` operations for this page (repairs may include `skipped` rows). */
   operations: OperationInsert[]
-  /** Whether another indexer page may exist after this one. */
-  hasMore: boolean
-  /** Offset to pass as `offset` on the next page. */
-  nextOffset: number
 }
 
-/** Options for {@link forEachDataSetPiecesPage}; pagination state is managed internally. */
-export type ForEachDataSetPiecesPageOptions = Omit<GetDataSetPiecesPageOptions, 'offset'>
-
 type PieceForOperation = {
   cid: string
-  metadata: Record<string, string> | null
-}
-
-/** Empty CID set for starting a paginated piece walk. */
-export function emptySeenCids(): Set<string> {
-  return new Set()
+  metadata: Record<string, string>
 }
 
 /**
- * Fetch one page of pieces for a provider at the repair block and map them to `add_piece` operations.
- *
- * Pieces are read from the indexer in stable dataset/piece order. CIDs are deduped globally so a
- * piece stored under multiple source datasets is queued once into the single IPFS repair dataset.
- * Alternate providers are resolved in one batch per page; operations without alternates are
- * inserted as `skipped` with a descriptive error.
+ * Map pieces to repair `add_piece` operations, probing the alternate providers for a copy of each.
  *
- * Pass `seenCids` and `nextOffset` from the prior result to continue pagination.
- *
- * @param options - Indexer connection, repair context, and optional pagination state.
- * @returns Operations for this page plus pagination cursors.
+ * Operations without a reachable alternate are returned as `skipped` with a descriptive error.
  */
-export async function getPiecesPage({
-  indexerDb,
-  providerId,
+async function piecesToRepairOperations({
+  pieces,
   repairId,
-  blockNumber,
-  limit = DEFAULT_PIECES_PAGE_SIZE,
-  offset = 0,
-  seenCids = emptySeenCids(),
-}: GetPiecesPageOptions): Promise<GetPiecesPageResult> {
-  const schema = indexerDb._.fullSchema
-  const rows = await indexerDb
-    .select({
-      cid: schema.pieces.cid,
-      metadata: schema.pieces.metadata,
-    })
-    .from(schema.pieces)
-    .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
-    .where(
-      and(
-        eq(schema.dataSets.providerId, providerId),
-        eq(schema.dataSets.deleted, false),
-        or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
-        eq(schema.pieces.removed, false)
-      )
-    )
-    .orderBy(asc(schema.pieces.dataSetId), asc(schema.pieces.pieceId))
-    .limit(limit)
-    .offset(offset)
-
+  alternateProviderUrls,
+}: {
+  pieces: PieceForOperation[]
+  repairId: number
+  alternateProviderUrls: string[]
+}): Promise<OperationInsert[]> {
   const now = Date.now()
-  const pieces: PieceForOperation[] = []
-
-  for (const { cid, metadata } of rows) {
-    // Same CID can appear on multiple source datasets; only repair it once.
-    if (seenCids.has(cid)) continue
-    seenCids.add(cid)
-
-    pieces.push({ cid, metadata })
-  }
-
-  // Resolve pull sources in one query per page; exclude the provider being repaired from alternates.
-  const providersByCid = await findProvidersByCid({
-    indexerDb,
-    cids: pieces.map((piece) => piece.cid),
-    blockNumber,
-  })
 
-  const operations: OperationInsert[] = await pMap(
+  return pMap(
     pieces,
     async ({ cid, metadata }) => {
-      const alternateProviders = providersByCid[cid]?.map((provider) => provider.serviceUrl) ?? []
       let skippedMessage = ''
       let validProvider: string | undefined
-      if (alternateProviders.length > 0) {
-        validProvider = await Piece.findPieceOnProviders(alternateProviders, Piece.from(cid))
+      if (alternateProviderUrls.length > 0) {
+        validProvider = await Piece.findPieceOnProviders(alternateProviderUrls, Piece.from(cid))
 
         if (!validProvider) {
-          skippedMessage = `Found ${alternateProviders.length} alternate providers, but none are valid. ${alternateProviders.join(', ')}`
+          skippedMessage = `Piece not found on any of ${alternateProviderUrls.length} probed providers`
         }
       } else {
         skippedMessage = `No alternate providers found`
@@ -163,129 +76,84 @@ export async function getPiecesPage({
         // Cannot pull without another replica; mark skipped up front so run skips these ops.
         status: validProvider ? 'pending' : 'skipped',
         cid,
-        metadata: metadata ?? {},
+        metadata,
         alternateProvider: validProvider ?? '',
         error: validProvider ? undefined : skippedMessage,
         createdAt: now,
         updatedAt: now,
-      }
+      } satisfies OperationInsert
     },
     { concurrency: 20 }
   )
-
-  // )
-
-  return {
-    operations,
-    // A full page means there may be more rows; a short page ends pagination.
-    hasMore: rows.length === limit,
-    nextOffset: offset + rows.length,
-    seenCids,
-  }
-}
-
-/**
- * Fetch one page of pieces for a specific dataset and map them to replication operations.
- *
- * Unlike repairs, replication preserves source dataset ordering and does not dedupe repeated CIDs.
- *
- * @param options - Indexer connection, dataset context, and optional pagination state.
- * @returns Operations for this page plus pagination cursors.
- */
-export async function getDataSetPiecesPage({
-  indexerDb,
-  dataSetId,
-  repairId,
-  sourceProviderUrl,
-  limit = DEFAULT_PIECES_PAGE_SIZE,
-  offset = 0,
-}: GetDataSetPiecesPageOptions): Promise<GetDataSetPiecesPageResult> {
-  const schema = indexerDb._.fullSchema
-  const rows = await indexerDb
-    .select({
-      cid: schema.pieces.cid,
-      metadata: schema.pieces.metadata,
-    })
-    .from(schema.pieces)
-    .where(and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false)))
-    .orderBy(asc(schema.pieces.pieceId))
-    .limit(limit)
-    .offset(offset)
-
-  const now = Date.now()
-  const operations: OperationInsert[] = rows.map(({ cid, metadata }) => ({
-    repairId,
-    type: 'add_piece',
-    status: 'pending',
-    cid,
-    metadata: metadata ?? {},
-    alternateProvider: sourceProviderUrl,
-    createdAt: now,
-    updatedAt: now,
-  }))
-
-  return {
-    operations,
-    hasMore: rows.length === limit,
-    nextOffset: offset + rows.length,
-  }
 }
 
 /**
- * Walk every page of `add_piece` operations for a provider, invoking `onPage` per batch.
+ * Walk every live piece of a provider at the repair block and map them to `add_piece` operations,
+ * invoking `onPage` per API page.
  *
- * Manages `offset` and `seenCids` across pages so callers only handle inserts.
+ * Pieces are walked newest dataset and piece first. CIDs are deduped globally so a piece stored
+ * under multiple source datasets is queued once into the single IPFS repair dataset. Each piece is
+ * looked up directly on the approved or endorsed providers, then the provider being repaired
+ * (`HEAD /piece/{cid}`), rather than in the API, so finding a pull source costs no API requests.
  *
- * @param options - Same inputs as {@link getPiecesPage} except pagination cursors.
- * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ * @param options - API client and repair context.
+ * @param onPage - Async handler for each page (e.g. batch insert into local DB).
  */
 export async function forEachPiecesPage(
-  options: ForEachPiecesPageOptions,
-  onPage: (page: GetPiecesPageResult) => Promise<void>
+  { api, providerId, repairId, blockNumber }: ForEachPiecesPageOptions,
+  onPage: (page: PiecesPage) => Promise<void>
 ): Promise<void> {
-  let offset = 0
-  let hasMore = true
-  let seenCids = emptySeenCids()
-
-  while (hasMore) {
-    const page = await getPiecesPage({
-      ...options,
-      offset,
-      seenCids,
-    })
-
-    await onPage(page)
-
-    offset = page.nextOffset
-    seenCids = page.seenCids
-    hasMore = page.hasMore
+  const activeDataSetIds = await getActiveDataSetIds({ api, providerId, blockNumber })
+  const alternateProviderUrls = await getAlternateProviderUrls({ api, repairProviderId: providerId })
+  const seenCids = new Set<string>()
+
+  for await (const rows of providerPiecePages(api, providerId, { removed: false })) {
+    const pieces: PieceForOperation[] = []
+    for (const { cid, metadata, dataSetId } of rows) {
+      if (!activeDataSetIds.has(dataSetId)) continue
+      // Same CID can appear on multiple source datasets; only repair it once.
+      if (cid == null || seenCids.has(cid)) continue
+      seenCids.add(cid)
+      pieces.push({ cid, metadata: toStringMetadata(metadata) })
+    }
+
+    await onPage({ operations: await piecesToRepairOperations({ pieces, repairId, alternateProviderUrls }) })
   }
 }
 
 /**
- * Walk every page of `add_piece` operations for a source dataset, invoking `onPage` per batch.
+ * Walk every live piece of a source dataset and map them to replication operations, invoking
+ * `onPage` per batch.
  *
- * Replication pagination intentionally has no CID dedupe state so duplicate pieces are preserved.
+ * Unlike repairs, replication preserves source dataset ordering (ascending piece ID) and does not
+ * dedupe repeated CIDs.
  *
- * @param options - Same inputs as {@link getDataSetPiecesPage} except pagination cursor.
- * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ * @param options - API client and dataset context.
+ * @param onPage - Async handler for each page (e.g. batch insert into local DB).
  */
 export async function forEachDataSetPiecesPage(
-  options: ForEachDataSetPiecesPageOptions,
-  onPage: (page: GetDataSetPiecesPageResult) => Promise<void>
+  { api, dataSetId, repairId, sourceProviderUrl }: ForEachDataSetPiecesPageOptions,
+  onPage: (page: PiecesPage) => Promise<void>
 ): Promise<void> {
-  let offset = 0
-  let hasMore = true
-
-  while (hasMore) {
-    const page = await getDataSetPiecesPage({
-      ...options,
-      offset,
-    })
-
-    await onPage(page)
+  // The API lists newest pieces first; load the whole dataset and reverse to keep source order.
+  const pieces = (await listDataSetPieces(api, dataSetId, { removed: false })).reverse()
+  const now = Date.now()
 
-    offset = page.nextOffset
-    hasMore = page.hasMore
+  for (let offset = 0; offset < pieces.length; offset += MAX_PAGE_SIZE) {
+    const operations: OperationInsert[] = []
+    for (const { cid, metadata } of pieces.slice(offset, offset + MAX_PAGE_SIZE)) {
+      if (cid == null) continue
+      operations.push({
+        repairId,
+        type: 'add_piece',
+        status: 'pending',
+        cid,
+        metadata: toStringMetadata(metadata),
+        alternateProvider: sourceProviderUrl,
+        createdAt: now,
+        updatedAt: now,
+      })
+    }
+    await onPage({ operations })
   }
 }

package/src/db/get-repair-provider.ts

@@ -1,47 +1,27 @@
-import { and, eq } from 'drizzle-orm'
+import type { Address } from 'viem'
+import { getProvider } from '../api.ts'
 import type { Context, RepairProvider } from '../types.ts'
 
-export interface GetRepairProviderOptions extends Pick<Context, 'indexerDb'> {
+export interface GetRepairProviderOptions extends Pick<Context, 'api'> {
   providerId: bigint
 }
 
 /**
  * Load an active provider by ID for use as a repair target.
  */
-export async function getRepairProvider({
-  indexerDb,
-  providerId,
-}: GetRepairProviderOptions): Promise<RepairProvider | null> {
-  const schema = indexerDb._.fullSchema
-  const [provider] = await indexerDb
-    .select({
-      providerId: schema.providers.providerId,
-      providerAddress: schema.providers.providerAddress,
-      name: schema.providers.name,
-      serviceUrl: schema.providers.serviceUrl,
-      approved: schema.providers.approved,
-      endorsed: schema.providers.endorsed,
-    })
-    .from(schema.providers)
-    .where(
-      and(
-        eq(schema.providers.providerId, providerId),
-        eq(schema.providers.providerActive, true),
-        eq(schema.providers.pdpProductActive, true)
-      )
-    )
-    .limit(1)
+export async function getRepairProvider({ api, providerId }: GetRepairProviderOptions): Promise<RepairProvider | null> {
+  const provider = await getProvider(api, providerId)
 
-  if (!provider?.providerAddress || !provider?.serviceUrl || !provider?.name) {
+  if (!provider?.active || !provider.pdpProductActive || !provider.serviceUrl || !provider.name) {
     return null
   }
 
   return {
-    providerId: provider.providerId,
-    providerAddress: provider.providerAddress,
+    providerId: BigInt(provider.providerId),
+    providerAddress: provider.address as Address,
     name: provider.name,
     serviceUrl: provider.serviceUrl,
-    approved: provider.approved,
-    endorsed: provider.endorsed,
+    approved: provider.approved ?? false,
+    endorsed: provider.endorsed ?? false,
   }
 }

package/src/db/repair-create.ts

@@ -18,7 +18,7 @@ export interface RepairCreateOptions extends Context {
  * @returns {Promise<number>} The ID of the created repair.
  */
 export async function repairCreate(options: RepairCreateOptions): Promise<number> {
-  const { indexerDb, localDb, repairProviderId, targetProviderId, client } = options
+  const { api, localDb, repairProviderId, targetProviderId, client } = options
   const localSchema = localDb._.fullSchema
   const now = Date.now()
   const blockNumber = await getBlockNumber(client)
@@ -34,7 +34,7 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
     throw new RepairCreationError('Target provider must differ from the provider being repaired')
   }
   const targetProvider = await getRepairProvider({
-    indexerDb,
+    api,
     providerId: targetProviderId,
   })
 
@@ -46,8 +46,10 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
   const [repair] = await localDb
     .insert(localSchema.repairs)
     .values({
+      kind: 'repair',
       repairProviderId,
       repairDataSetId: null,
+      sourceProviderUrl: null,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
       targetDataSetId: null,
@@ -66,7 +68,7 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
   const seenCids = new Set<string>()
   await forEachPiecesPage(
     {
-      indexerDb,
+      api,
       providerId: repairProviderId,
       repairId: repair.id,
       blockNumber,

package/src/db/replicate-create.ts

@@ -1,6 +1,6 @@
 import { taskLog } from '@clack/prompts'
-import { eq } from 'drizzle-orm'
 import { getBlockNumber } from 'viem/actions'
+import { getDataSet } from '../api.ts'
 import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
 import type { Context } from '../types.ts'
 import { forEachDataSetPiecesPage } from './get-pieces.ts'
@@ -18,8 +18,7 @@ export interface ReplicateCreateOptions extends Context {
  * @returns The ID of the created local job.
  */
 export async function replicateCreate(options: ReplicateCreateOptions): Promise<number> {
-  const { indexerDb, localDb, dataSetId, targetProviderId, client } = options
-  const indexerSchema = indexerDb._.fullSchema
+  const { api, localDb, dataSetId, targetProviderId, client } = options
   const localSchema = localDb._.fullSchema
   const now = Date.now()
   const blockNumber = await getBlockNumber(client)
@@ -30,35 +29,28 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
     retainLog: true,
   })
 
-  const [sourceDataSet] = await indexerDb
-    .select({
-      dataSetId: indexerSchema.dataSets.dataSetId,
-      providerId: indexerSchema.dataSets.providerId,
-      deleted: indexerSchema.dataSets.deleted,
-    })
-    .from(indexerSchema.dataSets)
-    .where(eq(indexerSchema.dataSets.dataSetId, dataSetId))
-    .limit(1)
+  const sourceDataSet = await getDataSet(api, dataSetId)
 
-  if (!sourceDataSet || sourceDataSet.deleted) {
+  if (!sourceDataSet || sourceDataSet.deleted || sourceDataSet.providerId == null) {
     throw new RepairCreationError(`Source dataset ${dataSetId} not found or deleted`)
   }
+  const sourceProviderId = BigInt(sourceDataSet.providerId)
 
-  if (targetProviderId === sourceDataSet.providerId) {
+  if (targetProviderId === sourceProviderId) {
     throw new RepairCreationError('Target provider must differ from the source dataset provider')
   }
 
   const sourceProvider = await getRepairProvider({
-    indexerDb,
-    providerId: sourceDataSet.providerId,
+    api,
+    providerId: sourceProviderId,
   })
 
   if (!sourceProvider) {
-    throw new RepairCreationError(`Source provider ${sourceDataSet.providerId} not found or inactive`)
+    throw new RepairCreationError(`Source provider ${sourceProviderId} not found or inactive`)
   }
 
   const targetProvider = await getRepairProvider({
-    indexerDb,
+    api,
     providerId: targetProviderId,
   })
 
@@ -69,8 +61,10 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
   const [repair] = await localDb
     .insert(localSchema.repairs)
     .values({
+      kind: 'replicate',
       repairProviderId: sourceProvider.providerId,
-      repairDataSetId: sourceDataSet.dataSetId,
+      repairDataSetId: dataSetId,
+      sourceProviderUrl: sourceProvider.serviceUrl,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
       targetDataSetId: null,
@@ -85,8 +79,8 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
   let totalOperations = 0
   await forEachDataSetPiecesPage(
     {
-      indexerDb,
-      dataSetId: sourceDataSet.dataSetId,
+      api,
+      dataSetId,
       repairId: repair.id,
       sourceProviderUrl: sourceProvider.serviceUrl,
     },