@filoz/repair-cli 0.0.1

package/src/db/get-pieces.ts

@@ -0,0 +1,189 @@
+import * as Piece from '@filoz/synapse-core/piece'
+import { and, asc, eq, isNull, lte, or } from 'drizzle-orm'
+import pMap from 'p-map'
+import type { OperationInsert } from '../local-schema.ts'
+import type { IndexerDatabase } from '../types.ts'
+import { getProvidersByCid } from './get-providers-by-cid.ts'
+
+/** Default page size when paginating pieces from the indexer. */
+export const DEFAULT_PIECES_PAGE_SIZE = 3000
+
+/** Options for fetching one page of `add_piece` operations for a repair. */
+export type GetPiecesPageOptions = {
+  indexerDb: IndexerDatabase
+  /** Source provider whose pieces are being repaired. */
+  providerId: bigint
+  /** Local repair row to attach operations to. */
+  repairId: number
+  /** Chain block number captured when the repair was created. */
+  blockNumber: bigint
+  /** Max indexer rows per page. Defaults to {@link DEFAULT_PIECES_PAGE_SIZE}. */
+  limit?: number
+  /** SQL offset for the indexer query. */
+  offset?: number
+  /**
+   * CIDs already emitted across prior pages. Pass the value returned from the previous call
+   * so the same CID is not queued twice when it appears in multiple source datasets.
+   */
+  seenCids?: Set<string>
+}
+
+/** Result of a single {@link getPiecesPage} call. */
+export type GetPiecesPageResult = {
+  /** `add_piece` operations ready to insert for this page (may include `skipped` rows). */
+  operations: OperationInsert[]
+  /** Whether another indexer page may exist after this one. */
+  hasMore: boolean
+  /** Offset to pass as `offset` on the next page. */
+  nextOffset: number
+  /** Updated dedupe set; pass into the next {@link getPiecesPage} call. */
+  seenCids: Set<string>
+}
+
+/** Options for {@link forEachPiecesPage}; pagination state is managed internally. */
+export type ForEachPiecesPageOptions = Omit<GetPiecesPageOptions, 'offset' | 'seenCids'>
+
+type PieceForOperation = {
+  cid: string
+  metadata: Record<string, string> | null
+}
+
+/** Empty CID set for starting a paginated piece walk. */
+export function emptySeenCids(): Set<string> {
+  return new Set()
+}
+
+/**
+ * Fetch one page of pieces for a provider at the repair block and map them to `add_piece` operations.
+ *
+ * Pieces are read from the indexer in stable dataset/piece order. CIDs are deduped globally so a
+ * piece stored under multiple source datasets is queued once into the single IPFS repair dataset.
+ * Alternate providers are resolved in one batch per page; operations without alternates are
+ * inserted as `skipped` with a descriptive error.
+ *
+ * Pass `seenCids` and `nextOffset` from the prior result to continue pagination.
+ *
+ * @param options - Indexer connection, repair context, and optional pagination state.
+ * @returns Operations for this page plus pagination cursors.
+ */
+export async function getPiecesPage({
+  indexerDb,
+  providerId,
+  repairId,
+  blockNumber,
+  limit = DEFAULT_PIECES_PAGE_SIZE,
+  offset = 0,
+  seenCids = emptySeenCids(),
+}: GetPiecesPageOptions): Promise<GetPiecesPageResult> {
+  const schema = indexerDb._.fullSchema
+  const rows = await indexerDb
+    .select({
+      cid: schema.pieces.cid,
+      metadata: schema.pieces.metadata,
+    })
+    .from(schema.pieces)
+    .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
+    .where(
+      and(
+        eq(schema.dataSets.providerId, providerId),
+        eq(schema.dataSets.deleted, false),
+        or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
+        eq(schema.pieces.removed, false)
+      )
+    )
+    .orderBy(asc(schema.pieces.dataSetId), asc(schema.pieces.pieceId))
+    .limit(limit)
+    .offset(offset)
+
+  const now = Date.now()
+  const pieces: PieceForOperation[] = []
+
+  for (const { cid, metadata } of rows) {
+    // Same CID can appear on multiple source datasets; only repair it once.
+    if (seenCids.has(cid)) continue
+    seenCids.add(cid)
+
+    pieces.push({ cid, metadata })
+  }
+
+  // Resolve pull sources in one query per page; exclude the provider being repaired from alternates.
+  const providersByCid = await getProvidersByCid({
+    indexerDb,
+    cids: pieces.map((piece) => piece.cid),
+    excludedProviderIds: [],
+    blockNumber,
+  })
+
+  const operations: OperationInsert[] = await pMap(
+    pieces,
+    async ({ cid, metadata }) => {
+      const alternateProviders = providersByCid[cid]?.map((provider) => provider.serviceUrl) ?? []
+      let skippedMessage = ''
+      let validProvider: string | undefined
+      if (alternateProviders.length > 0) {
+        validProvider = await Piece.findPieceOnProviders(alternateProviders, Piece.from(cid))
+
+        if (!validProvider) {
+          skippedMessage = `Found ${alternateProviders.length} alternate providers, but none are valid. ${alternateProviders.join(', ')}`
+        }
+      } else {
+        skippedMessage = `No alternate providers found`
+      }
+
+      return {
+        repairId,
+        type: 'add_piece',
+        // Cannot pull without another replica; mark skipped up front so run skips these ops.
+        status: validProvider ? 'pending' : 'skipped',
+        cid,
+        metadata: metadata ?? {},
+        alternateProvider: validProvider ?? '',
+        error: validProvider ? undefined : skippedMessage,
+        createdAt: now,
+        updatedAt: now,
+      }
+    },
+    { concurrency: 20 }
+  )
+
+  // )
+
+  return {
+    operations,
+    // A full page means there may be more rows; a short page ends pagination.
+    hasMore: rows.length === limit,
+    nextOffset: offset + rows.length,
+    seenCids,
+  }
+}
+
+/**
+ * Walk every page of `add_piece` operations for a provider, invoking `onPage` per batch.
+ *
+ * Manages `offset` and `seenCids` across pages so callers only handle inserts.
+ *
+ * @param options - Same inputs as {@link getPiecesPage} except pagination cursors.
+ * @param onPage - Async handler for each page result (e.g. batch insert into local DB).
+ */
+export async function forEachPiecesPage(
+  options: ForEachPiecesPageOptions,
+  onPage: (page: GetPiecesPageResult) => Promise<void>
+): Promise<void> {
+  let offset = 0
+  let hasMore = true
+  let seenCids = emptySeenCids()
+
+  while (hasMore) {
+    const page = await getPiecesPage({
+      ...options,
+      offset,
+      seenCids,
+    })
+
+    await onPage(page)
+
+    offset = page.nextOffset
+    seenCids = page.seenCids
+    hasMore = page.hasMore
+  }
+}

package/src/db/get-providers-by-cid.ts

@@ -0,0 +1,75 @@
+import { and, asc, eq, inArray, isNull, lte, notInArray, or } from 'drizzle-orm'
+import type { IndexerDatabase, RepairProvider } from '../types.ts'
+
+export type GetProvidersByCidOptions = {
+  indexerDb: IndexerDatabase
+  cids: readonly string[]
+  excludedProviderIds: readonly bigint[]
+  blockNumber: bigint
+}
+
+/**
+ * Map of piece CID to providers that store that CID at the repair block.
+ */
+export type ProvidersByCid = Record<string, RepairProvider[]>
+
+/**
+ * Find alternate providers for each CID, excluding the given provider IDs.
+ *
+ * Deleted datasets and removed pieces are ignored. Only approved or endorsed providers are
+ * included. Every requested CID is present in the result; CIDs with no alternate providers
+ * map to an empty array.
+ */
+export async function getProvidersByCid({
+  indexerDb,
+  cids,
+  excludedProviderIds,
+  blockNumber,
+}: GetProvidersByCidOptions): Promise<ProvidersByCid> {
+  const schema = indexerDb._.fullSchema
+  const providersByCid = Object.fromEntries(cids.map((cid) => [cid, []])) as ProvidersByCid
+  if (cids.length === 0) return providersByCid
+
+  const filters = [
+    inArray(schema.pieces.cid, [...cids]),
+    eq(schema.dataSets.deleted, false),
+    or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
+    eq(schema.pieces.removed, false),
+    // or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)),
+  ]
+  if (excludedProviderIds.length > 0) {
+    filters.push(notInArray(schema.dataSets.providerId, [...excludedProviderIds]))
+  }
+
+  // Join through datasets because providers own datasets, while pieces only reference dataset IDs.
+  const rows = await indexerDb
+    .selectDistinct({
+      cid: schema.pieces.cid,
+      providerId: schema.providers.providerId,
+      providerAddress: schema.providers.providerAddress,
+      name: schema.providers.name,
+      serviceUrl: schema.providers.serviceUrl,
+      approved: schema.providers.approved,
+      endorsed: schema.providers.endorsed,
+    })
+    .from(schema.pieces)
+    .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
+    .innerJoin(schema.providers, eq(schema.dataSets.providerId, schema.providers.providerId))
+    .where(and(...filters))
+    .orderBy(asc(schema.pieces.cid), asc(schema.providers.providerId))
+
+  for (const { cid, ...provider } of rows) {
+    if (provider.providerAddress && provider.serviceUrl && provider.name) {
+      providersByCid[cid]?.push({
+        providerId: provider.providerId,
+        providerAddress: provider.providerAddress,
+        name: provider.name,
+        serviceUrl: provider.serviceUrl,
+        approved: provider.approved,
+        endorsed: provider.endorsed,
+      })
+    }
+  }
+
+  return providersByCid
+}

package/src/db/get-repair-dataset.ts

@@ -0,0 +1,44 @@
+import { and, asc, eq, isNull } from 'drizzle-orm'
+import type { Address } from 'viem'
+import type { IndexerDatabase } from '../types.ts'
+import { EARLY_REPAIR_SOURCE } from '../utils.ts'
+
+export type GetRepairDatasetOptions = {
+  indexerDb: IndexerDatabase
+  providerId: bigint
+  payer: Address
+}
+
+/**
+ * Find one IPFS-enabled repair dataset for a provider at the repair block, if it exists.
+ *
+ * When multiple datasets match, the lowest `dataSetId` is returned.
+ */
+export async function getRepairDataset({
+  indexerDb,
+  providerId,
+  payer,
+}: GetRepairDatasetOptions): Promise<bigint | null> {
+  const schema = indexerDb._.fullSchema
+
+  const [row] = await indexerDb
+    .select({
+      dataSetId: schema.dataSets.dataSetId,
+    })
+    .from(schema.dataSets)
+    .where(
+      and(
+        eq(schema.dataSets.providerId, providerId),
+        eq(schema.dataSets.deleted, false),
+        isNull(schema.dataSets.pdpEndEpoch),
+        eq(schema.dataSets.payer, payer.toLowerCase()),
+        eq(schema.dataSets.source, EARLY_REPAIR_SOURCE),
+        eq(schema.dataSets.withCdn, false),
+        eq(schema.dataSets.withIpfsIndexing, true)
+      )
+    )
+    .orderBy(asc(schema.dataSets.dataSetId))
+    .limit(1)
+
+  return row.dataSetId ?? null
+}

package/src/db/get-repair-provider.ts

@@ -0,0 +1,47 @@
+import { and, eq } from 'drizzle-orm'
+import type { Context, RepairProvider } from '../types.ts'
+
+export interface GetRepairProviderOptions extends Pick<Context, 'indexerDb'> {
+  providerId: bigint
+}
+
+/**
+ * Load an active provider by ID for use as a repair target.
+ */
+export async function getRepairProvider({
+  indexerDb,
+  providerId,
+}: GetRepairProviderOptions): Promise<RepairProvider | null> {
+  const schema = indexerDb._.fullSchema
+  const [provider] = await indexerDb
+    .select({
+      providerId: schema.providers.providerId,
+      providerAddress: schema.providers.providerAddress,
+      name: schema.providers.name,
+      serviceUrl: schema.providers.serviceUrl,
+      approved: schema.providers.approved,
+      endorsed: schema.providers.endorsed,
+    })
+    .from(schema.providers)
+    .where(
+      and(
+        eq(schema.providers.providerId, providerId),
+        eq(schema.providers.providerActive, true),
+        eq(schema.providers.pdpProductActive, true)
+      )
+    )
+    .limit(1)
+
+  if (!provider?.providerAddress || !provider?.serviceUrl || !provider?.name) {
+    return null
+  }
+
+  return {
+    providerId: provider.providerId,
+    providerAddress: provider.providerAddress,
+    name: provider.name,
+    serviceUrl: provider.serviceUrl,
+    approved: provider.approved,
+    endorsed: provider.endorsed,
+  }
+}

package/src/db/get-target-dataset.ts

@@ -0,0 +1,47 @@
+import { getDataSet } from '@filoz/synapse-core/warm-storage'
+import { eq } from 'drizzle-orm'
+import { MissingRepairDataSetError, RepairNotFoundError } from '../error.ts'
+import type { LocalDatabase, WalletClient } from '../types.ts'
+
+const targetDatasetCache = new Map<number, getDataSet.OutputType>()
+
+/**
+ * Get the single IPFS-enabled target dataset for a repair.
+ *
+ * @param options - The options for getting the target dataset.
+ */
+export async function getTargetDataset({
+  localDb,
+  repairId,
+  client,
+}: {
+  localDb: LocalDatabase
+  repairId: number
+  client: WalletClient
+}) {
+  const cached = targetDatasetCache.get(repairId)
+  if (cached) {
+    return cached
+  }
+
+  const repair = await localDb.query.repairs.findFirst({
+    where: eq(localDb._.fullSchema.repairs.id, repairId),
+    columns: { targetDataSetId: true },
+  })
+  if (!repair) {
+    throw new RepairNotFoundError(repairId)
+  }
+
+  if (repair.targetDataSetId == null) {
+    throw new MissingRepairDataSetError()
+  }
+
+  const dataSet = await getDataSet(client, { dataSetId: repair.targetDataSetId })
+  if (!dataSet) {
+    throw new MissingRepairDataSetError()
+  }
+
+  targetDatasetCache.set(repairId, dataSet)
+
+  return dataSet
+}

package/src/db/repair-create.ts

@@ -0,0 +1,101 @@
+import { taskLog } from '@clack/prompts'
+import { getBlockNumber } from 'viem/actions'
+import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
+import type { Context } from '../types.ts'
+import { forEachPiecesPage } from './get-pieces.ts'
+import { getRepairProvider } from './get-repair-provider.ts'
+
+export interface RepairCreateOptions extends Context {
+  repairProviderId: bigint
+  targetProviderId: bigint
+}
+
+/**
+ * Prepare a repair by selecting a target provider, creating the repair row, and
+ * inserting pending dataset and piece operations.
+ *
+ * @param {RepairCreateOptions} options - The options for creating a repair.
+ * @returns {Promise<number>} The ID of the created repair.
+ */
+export async function repairCreate(options: RepairCreateOptions): Promise<number> {
+  const { indexerDb, localDb, repairProviderId, targetProviderId, client } = options
+  const localSchema = localDb._.fullSchema
+  const now = Date.now()
+  const blockNumber = await getBlockNumber(client)
+
+  const log = taskLog({
+    title: 'Creating repair',
+    limit: 10,
+    retainLog: true,
+  })
+
+  // Load the explicit target provider.
+  if (targetProviderId === repairProviderId) {
+    throw new RepairCreationError('Target provider must differ from the provider being repaired')
+  }
+  const targetProvider = await getRepairProvider({
+    indexerDb,
+    providerId: targetProviderId,
+  })
+
+  if (!targetProvider) {
+    throw new NoAlternateProviderError(targetProviderId)
+  }
+
+  // Create the repair row
+  const [repair] = await localDb
+    .insert(localSchema.repairs)
+    .values({
+      repairProviderId,
+      targetProviderId: targetProvider.providerId,
+      targetProviderUrl: targetProvider.serviceUrl,
+      targetDataSetId: null,
+      blockNumber,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .returning({ id: localSchema.repairs.id })
+
+  if (!repair) throw new RepairCreationError()
+
+  // Add the pieces to the repair
+  let totalOperations = 0
+  let totalPendingOperations = 0
+  let totalSkippedOperations = 0
+  const seenCids = new Set<string>()
+  await forEachPiecesPage(
+    {
+      indexerDb,
+      providerId: repairProviderId,
+      repairId: repair.id,
+      blockNumber,
+    },
+    async (page) => {
+      for (const operation of page.operations) {
+        if (seenCids.has(operation.cid)) {
+          continue
+        }
+        seenCids.add(operation.cid)
+      }
+      const pendingOperations = page.operations.filter((operation) => operation.status === 'pending').length
+      const skippedOperations = page.operations.filter((operation) => operation.status === 'skipped').length
+      totalOperations += page.operations.length
+      totalPendingOperations += pendingOperations
+      totalSkippedOperations += skippedOperations
+
+      if (page.operations.length > 0) {
+        await localDb.insert(localSchema.operations).values(page.operations)
+      }
+
+      log.message(
+        `Inserted ${page.operations.length} operations (${pendingOperations} pending, ${skippedOperations} skipped)`
+      )
+    }
+  )
+
+  log.success(
+    `Created repair ${repair.id} with ${totalOperations} operations (${totalPendingOperations} pending, ${totalSkippedOperations} skipped)`,
+    { showLog: true }
+  )
+  return repair.id
+}

package/src/db/repair-delete.ts

@@ -0,0 +1,39 @@
+import { eq } from 'drizzle-orm'
+import * as localSchema from '../local-schema.ts'
+import type { LocalDatabase } from '../types.ts'
+
+export type RepairDeleteOptions = {
+  localDb: LocalDatabase
+  repairId: number
+}
+
+export type RepairDeleteResult = {
+  deleted: boolean
+  operationsDeleted: number
+}
+
+/**
+ * Delete a repair and all of its operations from the local database.
+ */
+export async function repairDelete({ localDb, repairId }: RepairDeleteOptions): Promise<RepairDeleteResult> {
+  const repair = await localDb.query.repairs.findFirst({
+    where: eq(localSchema.repairs.id, repairId),
+    columns: { id: true },
+    with: {
+      operations: {
+        columns: { id: true },
+      },
+    },
+  })
+
+  if (!repair) {
+    return { deleted: false, operationsDeleted: 0 }
+  }
+
+  const operationsDeleted = await localDb
+    .delete(localSchema.operations)
+    .where(eq(localSchema.operations.repairId, repairId))
+  await localDb.delete(localSchema.repairs).where(eq(localSchema.repairs.id, repairId))
+
+  return { deleted: true, operationsDeleted: operationsDeleted.rowsAffected }
+}

package/src/db/repair-update.ts

@@ -0,0 +1,20 @@
+import { eq } from 'drizzle-orm'
+import type { RepairUpdate } from '../local-schema.ts'
+import * as localSchema from '../local-schema.ts'
+import type { LocalDatabase } from '../types.ts'
+
+export type RepairUpdateOptions = {
+  localDb: LocalDatabase
+  repairId: number
+  status?: localSchema.RepairStatus
+  targetDataSetId?: bigint | null
+}
+
+export async function repairUpdate({ localDb, repairId, status, targetDataSetId }: RepairUpdateOptions) {
+  const update: RepairUpdate = {
+    updatedAt: Date.now(),
+  }
+  if (status) update.status = status
+  if (targetDataSetId !== undefined) update.targetDataSetId = targetDataSetId
+  await localDb.update(localSchema.repairs).set(update).where(eq(localSchema.repairs.id, repairId))
+}

package/src/db/sync-pieces-onchain.ts

@@ -0,0 +1,53 @@
+import { and, eq, inArray } from 'drizzle-orm'
+import type { OperationInsert, OperationSelect } from '../local-schema.ts'
+import type { IndexerDatabase, LocalDatabase } from '../types.ts'
+import { upsertOperations } from './upsert-operations.ts'
+
+export type SyncPiecesOnchainOptions = {
+  indexerDb: IndexerDatabase
+  localDb: LocalDatabase
+  dataSetId: bigint
+  cidToOperation: Map<string, OperationSelect>
+}
+
+/**
+ * Sync pieces onchain to avoid duplicates.
+ */
+export async function syncPiecesOnchain({ indexerDb, localDb, dataSetId, cidToOperation }: SyncPiecesOnchainOptions) {
+  const cids = Array.from(cidToOperation.keys())
+  const schema = indexerDb._.fullSchema
+  let completedOperations = 0
+  const rows = await indexerDb
+    .select({ cid: schema.pieces.cid })
+    .from(schema.pieces)
+    .where(
+      and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false), inArray(schema.pieces.cid, cids))
+    )
+
+  const existingCids = new Set<string>()
+  const completedOperation: OperationInsert[] = []
+
+  for (const row of rows) {
+    const operation = cidToOperation.get(row.cid)
+    if (!operation) {
+      continue
+    }
+    completedOperation.push({
+      ...operation,
+      status: 'completed',
+      error: null,
+    })
+    existingCids.add(row.cid)
+    cidToOperation.delete(row.cid)
+    completedOperations++
+  }
+
+  if (completedOperation.length > 0) {
+    await upsertOperations({
+      localDb,
+      operations: completedOperation,
+    })
+  }
+
+  return completedOperations
+}

package/src/db/update-operation.ts

@@ -0,0 +1,26 @@
+import { eq } from 'drizzle-orm'
+import * as localSchema from '../local-schema.ts'
+import type { LocalDatabase } from '../types.ts'
+
+export type UpdateOperationOptions = {
+  localDb: LocalDatabase
+  operationId: number
+  status: localSchema.OperationStatus
+  result?: localSchema.OperationResult | null
+  error?: string | null
+}
+
+/**
+ * Updates an operation in the database.
+ */
+export async function updateOperation({ localDb, operationId, status, result, error }: UpdateOperationOptions) {
+  await localDb
+    .update(localSchema.operations)
+    .set({
+      status,
+      result,
+      error: error ?? null,
+      updatedAt: Date.now(),
+    })
+    .where(eq(localSchema.operations.id, operationId))
+}

package/src/db/upsert-operations.ts

@@ -0,0 +1,23 @@
+import type { OperationInsert } from '../local-schema.ts'
+import * as localSchema from '../local-schema.ts'
+import type { LocalDatabase } from '../types.ts'
+import { buildConflictUpdateColumns } from '../utils.ts'
+
+export type UpsertOperationsOptions = {
+  localDb: LocalDatabase
+  operations: OperationInsert[]
+}
+
+/**
+ * Upserts operations in the database.
+ */
+export async function upsertOperations({ localDb, operations }: UpsertOperationsOptions) {
+  const now = Date.now()
+  await localDb
+    .insert(localDb._.fullSchema.operations)
+    .values(operations.map((operation) => ({ ...operation, updatedAt: now })))
+    .onConflictDoUpdate({
+      target: localDb._.fullSchema.operations.id,
+      set: buildConflictUpdateColumns(localSchema.operations, ['status', 'error', 'updatedAt']),
+    })
+}