@filoz/repair-cli 0.4.0 → 1.1.0

package/src/commands/setup.ts

@@ -3,21 +3,16 @@ import { Cli, z } from 'incur'
 import path from 'path'
 import type { Hash } from 'viem'
 import { privateKeyToAccount } from 'viem/accounts'
+import { DEFAULT_API_URL } from '../api.ts'
 import { config, createLocalDatabase, globalOptions, migrateLocalDatabase } from '../utils.ts'
 
-function validatePostgresUrl(value: string) {
-  let url: URL
+function validateHttpUrl(value: string) {
   try {
-    url = new URL(value)
+    const url = new URL(value)
+    return url.protocol === 'http:' || url.protocol === 'https:'
   } catch {
     return false
   }
-
-  if (url.protocol !== 'postgresql:' && url.protocol !== 'postgres:') {
-    return false
-  }
-
-  return Boolean(url.hostname && url.username && url.password && url.pathname.length > 1)
 }
 
 export const setup = Cli.create('setup', {
@@ -44,35 +39,18 @@ export const setup = Cli.create('setup', {
         })
       }
 
-      // Indexer URL
-      const indexerMainnetUrl = await p.text({
-        message: 'Enter your Mainnet Indexer Postgres URL',
-        validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
-          }
-        },
-        initialValue: config.get('indexerMainnetUrl'),
-        withGuide: false,
-      })
-      if (p.isCancel(indexerMainnetUrl)) {
-        return c.error({
-          code: 'SETUP_CANCELLED',
-          message: 'Setup cancelled',
-          retryable: false,
-        })
-      }
-      const indexerCalibrationUrl = await p.text({
-        message: 'Enter your Calibration Indexer Postgres URL',
+      // API URL
+      const apiUrl = await p.text({
+        message: 'Enter the Filecoin API URL',
         validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
+          if (!value || !validateHttpUrl(value)) {
+            return `Invalid http(s) URL!`
           }
         },
-        initialValue: config.get('indexerCalibrationUrl'),
+        initialValue: config.get('apiUrl') ?? DEFAULT_API_URL,
         withGuide: false,
       })
-      if (p.isCancel(indexerCalibrationUrl)) {
+      if (p.isCancel(apiUrl)) {
         return c.error({
           code: 'SETUP_CANCELLED',
           message: 'Setup cancelled',
@@ -129,8 +107,7 @@ export const setup = Cli.create('setup', {
 
       // Set config
       config.set('privateKey', pk)
-      config.set('indexerMainnetUrl', indexerMainnetUrl)
-      config.set('indexerCalibrationUrl', indexerCalibrationUrl)
+      config.set('apiUrl', apiUrl)
       config.set('chainId', chainId)
       config.set('dbPath', dbPath)
       config.set('source', source)

package/src/db/dedupe-cids.ts

@@ -1,37 +1,20 @@
-import { and, eq, inArray } from 'drizzle-orm'
 import type { OperationSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase } from '../types.ts'
+import type { LocalDatabase } from '../types.ts'
 import { upsertOperations } from './upsert-operations.ts'
 
-export type SyncPiecesOnchainOptions = {
-  indexerDb: IndexerDatabase
+export type DedupeCidsOptions = {
   localDb: LocalDatabase
-  dataSetId: bigint
+  /** CIDs already in the target repair dataset. */
+  existingCids: ReadonlySet<string>
   operations: OperationSelect[]
 }
 
 /**
- * Dedupe operations by CID on the target repair dataset.
+ * Mark operations whose CID is already in the target repair dataset as completed.
  * Returns operations that are not onchain.
  */
-export async function dedupeCids({ indexerDb, localDb, dataSetId, operations }: SyncPiecesOnchainOptions) {
-  const cids = operations.map((operation) => operation.cid)
-  const schema = indexerDb._.fullSchema
-
-  const rows = await indexerDb
-    .select({ cid: schema.pieces.cid })
-    .from(schema.pieces)
-    .where(
-      and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false), inArray(schema.pieces.cid, cids))
-    )
-
-  const existingCids = new Set<string>()
-  const completedOperations: OperationSelect[] = []
-
-  for (const row of rows) {
-    completedOperations.push(...operations.filter((operation) => operation.cid === row.cid))
-    existingCids.add(row.cid)
-  }
+export async function dedupeCids({ localDb, existingCids, operations }: DedupeCidsOptions) {
+  const completedOperations = operations.filter((operation) => existingCids.has(operation.cid))
 
   if (completedOperations.length > 0) {
     await upsertOperations({

package/src/db/find-repair-dataset.ts

@@ -1,49 +1,38 @@
-import { and, asc, eq, isNull } from 'drizzle-orm'
 import type { Address } from 'viem'
-import type { IndexerDatabase } from '../types.ts'
+import { listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
 
 export type FindRepairDatasetOptions = {
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   providerId: bigint
   payer: Address
   source: string
 }
 
 /**
- * Find one IPFS-enabled repair dataset for a provider at the repair block, if it exists.
+ * Find one live IPFS-enabled repair dataset for a provider, if it exists.
  *
  * When multiple datasets match, the lowest `dataSetId` is returned.
  */
 export async function findRepairDataset({
-  indexerDb,
+  api,
   providerId,
   payer,
   source,
 }: FindRepairDatasetOptions): Promise<bigint | null> {
-  const schema = indexerDb._.fullSchema
+  const dataSets = await listDataSets(api, {
+    owner: payer.toLowerCase(),
+    providerId,
+    deleted: false,
+    withCdn: false,
+  })
 
-  const result = await indexerDb
-    .select({
-      dataSetId: schema.dataSets.dataSetId,
-    })
-    .from(schema.dataSets)
-    .where(
-      and(
-        eq(schema.dataSets.providerId, providerId),
-        eq(schema.dataSets.deleted, false),
-        isNull(schema.dataSets.pdpEndEpoch),
-        eq(schema.dataSets.payer, payer.toLowerCase()),
-        eq(schema.dataSets.source, source),
-        eq(schema.dataSets.withCdn, false),
-        eq(schema.dataSets.withIpfsIndexing, true)
-      )
-    )
-    .orderBy(asc(schema.dataSets.dataSetId))
-    .limit(1)
-
-  if (result.length === 0) {
-    return null
+  let dataSetId: bigint | null = null
+  for (const dataSet of dataSets) {
+    if (dataSet.pdpEndEpoch != null || dataSet.source !== source || !dataSet.withIpfsIndexing) continue
+    const id = BigInt(dataSet.dataSetId)
+    if (dataSetId == null || id < dataSetId) dataSetId = id
   }
 
-  return result[0].dataSetId
+  return dataSetId
 }

package/src/db/get-active-data-set-ids.ts

@@ -0,0 +1,29 @@
+import { type DataSet, listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
+
+/**
+ * Whether a data set's pieces count at the repair block: not deleted, and either not terminated
+ * (no PDP end epoch) or terminated with an end epoch after `blockNumber`, so the provider still
+ * has to prove it.
+ */
+export function isDataSetActiveAt(dataSet: Pick<DataSet, 'deleted' | 'pdpEndEpoch'>, blockNumber: bigint): boolean {
+  return !dataSet.deleted && (dataSet.pdpEndEpoch == null || BigInt(dataSet.pdpEndEpoch) > blockNumber)
+}
+
+/**
+ * IDs of a provider's data sets that are active at the repair block (see {@link isDataSetActiveAt}).
+ */
+export async function getActiveDataSetIds({
+  api,
+  providerId,
+  blockNumber,
+}: {
+  api: IndexerApi
+  providerId: bigint
+  blockNumber: bigint
+}): Promise<Set<string>> {
+  const dataSets = await listDataSets(api, { providerId, deleted: false })
+  return new Set(
+    dataSets.filter((dataSet) => isDataSetActiveAt(dataSet, blockNumber)).map((dataSet) => dataSet.dataSetId)
+  )
+}

package/src/db/get-alternate-provider-urls.ts

@@ -0,0 +1,43 @@
+import { listProviders, type Provider } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
+
+/** Rank endorsed providers first, then approved, then by ascending ID. */
+function compareProviders(a: Provider, b: Provider): number {
+  if (a.endorsed !== b.endorsed) return a.endorsed ? -1 : 1
+  if (a.approved !== b.approved) return a.approved ? -1 : 1
+  const x = BigInt(a.providerId)
+  const y = BigInt(b.providerId)
+  return x < y ? -1 : x > y ? 1 : 0
+}
+
+/**
+ * Service URLs of the providers to probe for a copy of a repaired piece: approved or endorsed
+ * providers with a service URL, then the provider being repaired as a last resort, so pieces only
+ * it stores can still be pulled while it serves them.
+ *
+ * Callers probe these with `findPieceOnProviders`, which keeps this order, so endorsed providers
+ * are preferred as pull sources and the repaired provider is used only when no other one serves
+ * the piece.
+ */
+export async function getAlternateProviderUrls({
+  api,
+  repairProviderId,
+}: {
+  api: IndexerApi
+  repairProviderId: bigint
+}): Promise<string[]> {
+  const providers = await listProviders(api)
+  const repairProviderUrl = providers.find((provider) => BigInt(provider.providerId) === repairProviderId)?.serviceUrl
+  const urls = providers
+    .filter(
+      (provider) =>
+        (provider.approved || provider.endorsed) &&
+        provider.serviceUrl &&
+        BigInt(provider.providerId) !== repairProviderId
+    )
+    .sort(compareProviders)
+    .map((provider) => provider.serviceUrl as string)
+
+  if (repairProviderUrl) urls.push(repairProviderUrl)
+  return urls
+}

package/src/db/get-pieces.ts

@@ -1,157 +1,70 @@
 import * as Piece from '@filoz/synapse-core/piece'
-import { and, asc, eq, isNull, lte, or } from 'drizzle-orm'
 import pMap from 'p-map'
+import { listDataSetPieces, MAX_PAGE_SIZE, providerPiecePages, toStringMetadata } from '../api.ts'
 import type { OperationInsert } from '../local-schema.ts'
-import type { IndexerDatabase } from '../types.ts'
-import { findProvidersByCid } from './find-providers-by-cid.ts'
+import type { IndexerApi } from '../types.ts'
+import { getActiveDataSetIds } from './get-active-data-set-ids.ts'
+import { getAlternateProviderUrls } from './get-alternate-provider-urls.ts'
 
-/** Default page size when paginating pieces from the indexer. */
-export const DEFAULT_PIECES_PAGE_SIZE = 3000
-
-/** Options for fetching one page of `add_piece` operations for a repair. */
-export type GetPiecesPageOptions = {
-  indexerDb: IndexerDatabase
+/** Options for walking `add_piece` operations for a repair. */
+export type ForEachPiecesPageOptions = {
+  api: IndexerApi
   /** Source provider whose pieces are being repaired. */
   providerId: bigint
   /** Local repair row to attach operations to. */
   repairId: number
   /** Chain block number captured when the repair was created. */
   blockNumber: bigint
-  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
-  limit?: number
-  /** SQL offset for the indexer query. */
-  offset?: number
-  /**
-   * CIDs already emitted across prior pages. Pass the value returned from the previous call
-   * so the same CID is not queued twice when it appears in multiple source datasets.
-   */
-  seenCids?: Set<string>
-}
-
-/** Result of a single {@link getPiecesPage} call. */
-export type GetPiecesPageResult = {
-  /** `add_piece` operations ready to insert for this page (may include `skipped` rows). */
-  operations: OperationInsert[]
-  /** Whether another indexer page may exist after this one. */
-  hasMore: boolean
-  /** Offset to pass as `offset` on the next page. */
-  nextOffset: number
-  /** Updated dedupe set; pass into the next {@link getPiecesPage} call. */
-  seenCids: Set<string>
 }
 
-/** Options for {@link forEachPiecesPage}; pagination state is managed internally. */
-export type ForEachPiecesPageOptions = Omit<GetPiecesPageOptions, 'offset' | 'seenCids'>
-
-/** Options for fetching one page of source-dataset pieces for replication. */
-export type GetDataSetPiecesPageOptions = {
-  indexerDb: IndexerDatabase
+/** Options for walking source-dataset pieces for replication. */
+export type ForEachDataSetPiecesPageOptions = {
+  api: IndexerApi
   /** Source dataset whose pieces are being replicated. */
   dataSetId: bigint
   /** Local job row to attach operations to. */
   repairId: number
   /** Provider URL that serves the source dataset pieces. */
   sourceProviderUrl: string
-  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
-  limit?: number
-  /** SQL offset for the indexer query. */
-  offset?: number
 }
 
-/** Result of a single {@link getDataSetPiecesPage} call. */
-export type GetDataSetPiecesPageResult = {
-  /** `add_piece` operations ready to insert for this page. */
+/** One page of operations ready to insert into the local database. */
+export type PiecesPage = {
+  /** `add_piece` operations for this page (repairs may include `skipped` rows). */
   operations: OperationInsert[]
-  /** Whether another indexer page may exist after this one. */
-  hasMore: boolean
-  /** Offset to pass as `offset` on the next page. */
-  nextOffset: number
 }
 
-/** Options for {@link forEachDataSetPiecesPage}; pagination state is managed internally. */
-export type ForEachDataSetPiecesPageOptions = Omit<GetDataSetPiecesPageOptions, 'offset'>
-
 type PieceForOperation = {
   cid: string
-  metadata: Record<string, string> | null
-}
-
-/** Empty CID set for starting a paginated piece walk. */
-export function emptySeenCids(): Set<string> {
-  return new Set()
+  metadata: Record<string, string>
 }
 
 /**
- * Fetch one page of pieces for a provider at the repair block and map them to `add_piece` operations.
- *
- * Pieces are read from the indexer in stable dataset/piece order. CIDs are deduped globally so a
- * piece stored under multiple source datasets is queued once into the single IPFS repair dataset.
- * Alternate providers are resolved in one batch per page; operations without alternates are
- * inserted as `skipped` with a descriptive error.
+ * Map pieces to repair `add_piece` operations, probing the alternate providers for a copy of each.
  *
- * Pass `seenCids` and `nextOffset` from the prior result to continue pagination.
- *
- * @param options - Indexer connection, repair context, and optional pagination state.
- * @returns Operations for this page plus pagination cursors.
+ * Operations without a reachable alternate are returned as `skipped` with a descriptive error.
  */
-export async function getPiecesPage({
-  indexerDb,
-  providerId,
+async function piecesToRepairOperations({
+  pieces,
   repairId,
-  blockNumber,
-  limit = DEFAULT_PIECES_PAGE_SIZE,
-  offset = 0,
-  seenCids = emptySeenCids(),
-}: GetPiecesPageOptions): Promise<GetPiecesPageResult> {
-  const schema = indexerDb._.fullSchema
-  const rows = await indexerDb
-    .select({
-      cid: schema.pieces.cid,
-      metadata: schema.pieces.metadata,
-    })
-    .from(schema.pieces)
-    .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
-    .where(
-      and(
-        eq(schema.dataSets.providerId, providerId),
-        eq(schema.dataSets.deleted, false),
-        or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
-        eq(schema.pieces.removed, false)
-      )
-    )
-    .orderBy(asc(schema.pieces.dataSetId), asc(schema.pieces.pieceId))
-    .limit(limit)
-    .offset(offset)
-
+  alternateProviderUrls,
+}: {
+  pieces: PieceForOperation[]
+  repairId: number
+  alternateProviderUrls: string[]
+}): Promise<OperationInsert[]> {
   const now = Date.now()
-  const pieces: PieceForOperation[] = []
-
-  for (const { cid, metadata } of rows) {
-    // Same CID can appear on multiple source datasets; only repair it once.
-    if (seenCids.has(cid)) continue
-    seenCids.add(cid)
-
-    pieces.push({ cid, metadata })
-  }
-
-  // Resolve pull sources in one query per page; exclude the provider being repaired from alternates.
-  const providersByCid = await findProvidersByCid({
-    indexerDb,
-    cids: pieces.map((piece) => piece.cid),
-    blockNumber,
-  })
 
-  const operations: OperationInsert[] = await pMap(
+  return pMap(
     pieces,
     async ({ cid, metadata }) => {
-      const alternateProviders = providersByCid[cid]?.map((provider) => provider.serviceUrl) ?? []
       let skippedMessage = ''
       let validProvider: string | undefined
-      if (alternateProviders.length > 0) {
-        validProvider = await Piece.findPieceOnProviders(alternateProviders, Piece.from(cid))
+      if (alternateProviderUrls.length > 0) {
+        validProvider = await Piece.findPieceOnProviders(alternateProviderUrls, Piece.from(cid))
 
         if (!validProvider) {
-          skippedMessage = `Found ${alternateProviders.length} alternate providers, but none are valid. ${alternateProviders.join(', ')}`
+          skippedMessage = `Piece not found on any of ${alternateProviderUrls.length} probed providers`
         }
       } else {
         skippedMessage = `No alternate providers found`
@@ -163,129 +76,84 @@ export async function getPiecesPage({
         // Cannot pull without another replica; mark skipped up front so run skips these ops.
         status: validProvider ? 'pending' : 'skipped',
         cid,
-        metadata: metadata ?? {},
+        metadata,
         alternateProvider: validProvider ?? '',
         error: validProvider ? undefined : skippedMessage,
         createdAt: now,
         updatedAt: now,
-      }
+      } satisfies OperationInsert
     },
     { concurrency: 20 }
   )
-
-  // )
-
-  return {
-    operations,
-    // A full page means there may be more rows; a short page ends pagination.
-    hasMore: rows.length === limit,
-    nextOffset: offset + rows.length,
-    seenCids,
-  }
-}
-
-/**
- * Fetch one page of pieces for a specific dataset and map them to replication operations.
- *
- * Unlike repairs, replication preserves source dataset ordering and does not dedupe repeated CIDs.
- *
- * @param options - Indexer connection, dataset context, and optional pagination state.
- * @returns Operations for this page plus pagination cursors.
- */
-export async function getDataSetPiecesPage({
-  indexerDb,
-  dataSetId,
-  repairId,
-  sourceProviderUrl,
-  limit = DEFAULT_PIECES_PAGE_SIZE,
-  offset = 0,
-}: GetDataSetPiecesPageOptions): Promise<GetDataSetPiecesPageResult> {
-  const schema = indexerDb._.fullSchema
-  const rows = await indexerDb
-    .select({
-      cid: schema.pieces.cid,
-      metadata: schema.pieces.metadata,
-    })
-    .from(schema.pieces)
-    .where(and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false)))
-    .orderBy(asc(schema.pieces.pieceId))
-    .limit(limit)
-    .offset(offset)
-
-  const now = Date.now()
-  const operations: OperationInsert[] = rows.map(({ cid, metadata }) => ({
-    repairId,
-    type: 'add_piece',
-    status: 'pending',
-    cid,
-    metadata: metadata ?? {},
-    alternateProvider: sourceProviderUrl,
-    createdAt: now,
-    updatedAt: now,
-  }))
-
-  return {
-    operations,
-    hasMore: rows.length === limit,
-    nextOffset: offset + rows.length,
-  }
 }
 
 /**
- * Walk every page of `add_piece` operations for a provider, invoking `onPage` per batch.
+ * Walk every live piece of a provider at the repair block and map them to `add_piece` operations,
+ * invoking `onPage` per API page.
  *
- * Manages `offset` and `seenCids` across pages so callers only handle inserts.
+ * Pieces are walked newest dataset and piece first. CIDs are deduped globally so a piece stored
+ * under multiple source datasets is queued once into the single IPFS repair dataset. Each piece is
+ * looked up directly on the approved or endorsed providers, then the provider being repaired
+ * (`HEAD /piece/{cid}`), rather than in the API, so finding a pull source costs no API requests.
  *
- * @param options - Same inputs as {@link getPiecesPage} except pagination cursors.
- * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ * @param options - API client and repair context.
+ * @param onPage - Async handler for each page (e.g. batch insert into local DB).
  */
 export async function forEachPiecesPage(
-  options: ForEachPiecesPageOptions,
-  onPage: (page: GetPiecesPageResult) => Promise<void>
+  { api, providerId, repairId, blockNumber }: ForEachPiecesPageOptions,
+  onPage: (page: PiecesPage) => Promise<void>
 ): Promise<void> {
-  let offset = 0
-  let hasMore = true
-  let seenCids = emptySeenCids()
-
-  while (hasMore) {
-    const page = await getPiecesPage({
-      ...options,
-      offset,
-      seenCids,
-    })
-
-    await onPage(page)
-
-    offset = page.nextOffset
-    seenCids = page.seenCids
-    hasMore = page.hasMore
+  const activeDataSetIds = await getActiveDataSetIds({ api, providerId, blockNumber })
+  const alternateProviderUrls = await getAlternateProviderUrls({ api, repairProviderId: providerId })
+  const seenCids = new Set<string>()
+
+  for await (const rows of providerPiecePages(api, providerId, { removed: false })) {
+    const pieces: PieceForOperation[] = []
+    for (const { cid, metadata, dataSetId } of rows) {
+      if (!activeDataSetIds.has(dataSetId)) continue
+      // Same CID can appear on multiple source datasets; only repair it once.
+      if (cid == null || seenCids.has(cid)) continue
+      seenCids.add(cid)
+      pieces.push({ cid, metadata: toStringMetadata(metadata) })
+    }
+
+    await onPage({ operations: await piecesToRepairOperations({ pieces, repairId, alternateProviderUrls }) })
   }
 }
 
 /**
- * Walk every page of `add_piece` operations for a source dataset, invoking `onPage` per batch.
+ * Walk every live piece of a source dataset and map them to replication operations, invoking
+ * `onPage` per batch.
  *
- * Replication pagination intentionally has no CID dedupe state so duplicate pieces are preserved.
+ * Unlike repairs, replication preserves source dataset ordering (ascending piece ID) and does not
+ * dedupe repeated CIDs.
  *
- * @param options - Same inputs as {@link getDataSetPiecesPage} except pagination cursor.
- * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ * @param options - API client and dataset context.
+ * @param onPage - Async handler for each page (e.g. batch insert into local DB).
  */
 export async function forEachDataSetPiecesPage(
-  options: ForEachDataSetPiecesPageOptions,
-  onPage: (page: GetDataSetPiecesPageResult) => Promise<void>
+  { api, dataSetId, repairId, sourceProviderUrl }: ForEachDataSetPiecesPageOptions,
+  onPage: (page: PiecesPage) => Promise<void>
 ): Promise<void> {
-  let offset = 0
-  let hasMore = true
-
-  while (hasMore) {
-    const page = await getDataSetPiecesPage({
-      ...options,
-      offset,
-    })
-
-    await onPage(page)
+  // The API lists newest pieces first; load the whole dataset and reverse to keep source order.
+  const pieces = (await listDataSetPieces(api, dataSetId, { removed: false })).reverse()
+  const now = Date.now()
 
-    offset = page.nextOffset
-    hasMore = page.hasMore
+  for (let offset = 0; offset < pieces.length; offset += MAX_PAGE_SIZE) {
+    const operations: OperationInsert[] = []
+    for (const { cid, metadata } of pieces.slice(offset, offset + MAX_PAGE_SIZE)) {
+      if (cid == null) continue
+      operations.push({
+        repairId,
+        type: 'add_piece',
+        status: 'pending',
+        cid,
+        metadata: toStringMetadata(metadata),
+        alternateProvider: sourceProviderUrl,
+        createdAt: now,
+        updatedAt: now,
+      })
+    }
+    await onPage({ operations })
   }
 }