@filoz/repair-cli 0.0.1

package/src/error.ts

@@ -0,0 +1,33 @@
+export class NoAlternateProviderError extends Error {
+  readonly providerId?: bigint
+
+  /**
+   * @param providerId - When set, the explicit target provider was not found or inactive.
+   */
+  constructor(providerId?: bigint) {
+    super(providerId == null ? 'No alternate provider found' : `Target provider ${providerId} not found or inactive`)
+    this.name = 'NoAlternateProviderError'
+    this.providerId = providerId
+  }
+}
+
+export class RepairCreationError extends Error {
+  constructor(message = 'Failed to create repair row') {
+    super(message)
+    this.name = 'RepairCreationError'
+  }
+}
+
+export class RepairNotFoundError extends Error {
+  constructor(repairId: number) {
+    super(`Repair ${repairId} not found`)
+    this.name = 'RepairNotFoundError'
+  }
+}
+
+export class MissingRepairDataSetError extends Error {
+  constructor() {
+    super('Missing repair dataset ID')
+    this.name = 'MissingRepairDataSetError'
+  }
+}

package/src/indexer-schema.ts

@@ -0,0 +1,77 @@
+import { relations } from 'drizzle-orm'
+import { bigint, boolean, index, jsonb, pgSchema, primaryKey, text } from 'drizzle-orm/pg-core'
+import type { Address } from 'viem'
+
+export type JsonRecord = Record<string, string>
+
+const schema = pgSchema('early-repair')
+
+export const providers = schema.table(
+  'providers',
+  {
+    providerId: bigint('provider_id', { mode: 'bigint' }).primaryKey(),
+    providerAddress: text('provider_address').$type<Address>(),
+    name: text('name'),
+    serviceUrl: text('service_url'),
+    providerActive: boolean('provider_active').notNull(),
+    pdpProductActive: boolean('pdp_product_active').notNull(),
+    approved: boolean('approved').notNull().default(false),
+    endorsed: boolean('endorsed').notNull().default(false),
+    createdAtBlock: bigint('created_at_block', { mode: 'bigint' }),
+    updatedAtBlock: bigint('updated_at_block', { mode: 'bigint' }).notNull(),
+  },
+  (table) => [index('providers_provider_address_idx').on(table.providerAddress)]
+)
+
+export const providersRelations = relations(providers, ({ many }) => ({
+  dataSets: many(dataSets),
+}))
+
+export const dataSets = schema.table(
+  'data_sets',
+  {
+    dataSetId: bigint('data_set_id', { mode: 'bigint' }).primaryKey(),
+    providerId: bigint('provider_id', { mode: 'bigint' }).notNull(),
+    metadata: jsonb('metadata').$type<JsonRecord | null>(),
+    payer: text('payer').notNull(),
+    source: text('source'),
+    withCdn: boolean('with_cdn').notNull(),
+    withIpfsIndexing: boolean('with_ipfs_indexing').notNull(),
+    pdpEndEpoch: bigint('pdp_end_epoch', { mode: 'bigint' }).notNull(),
+    deleted: boolean('deleted').notNull(),
+    createdAtBlock: bigint('created_at_block', { mode: 'bigint' }).notNull(),
+    updatedAtBlock: bigint('updated_at_block', { mode: 'bigint' }).notNull(),
+  },
+  (table) => [index('data_sets_provider_id_idx').on(table.providerId)]
+)
+
+export const dataSetsRelations = relations(dataSets, ({ one, many }) => ({
+  provider: one(providers, {
+    fields: [dataSets.providerId],
+    references: [providers.providerId],
+  }),
+  pieces: many(pieces),
+}))
+
+export const pieces = schema.table(
+  'pieces',
+  {
+    dataSetId: bigint('data_set_id', { mode: 'bigint' }).notNull(),
+    pieceId: bigint('piece_id', { mode: 'bigint' }).notNull(),
+    cid: text('cid').notNull(),
+    rawSize: bigint('raw_size', { mode: 'bigint' }).notNull(),
+    metadata: jsonb('metadata').$type<JsonRecord | null>(),
+    removed: boolean('removed').notNull(),
+    addedAtBlock: bigint('added_at_block', { mode: 'bigint' }).notNull(),
+    removedAtBlock: bigint('removed_at_block', { mode: 'bigint' }),
+    updatedAtBlock: bigint('updated_at_block', { mode: 'bigint' }).notNull(),
+  },
+  (table) => [primaryKey({ columns: [table.dataSetId, table.pieceId] }), index('pieces_cid_idx').on(table.cid)]
+)
+
+export const piecesRelations = relations(pieces, ({ one }) => ({
+  dataSet: one(dataSets, {
+    fields: [pieces.dataSetId],
+    references: [dataSets.dataSetId],
+  }),
+}))

package/src/local-schema.ts

@@ -0,0 +1,91 @@
+import type { MetadataObject } from '@filoz/synapse-core'
+import type * as SP from '@filoz/synapse-core/sp'
+import { relations } from 'drizzle-orm'
+import type { AnySQLiteColumn } from 'drizzle-orm/sqlite-core'
+import * as t from 'drizzle-orm/sqlite-core'
+import { customType, sqliteTable as table } from 'drizzle-orm/sqlite-core'
+import * as Json from 'iso-base/json'
+
+export type RepairStatus = 'pending' | 'completed' | 'failed'
+export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped'
+export type OperationType = 'create_dataset' | 'add_piece'
+
+export type OperationResult = Omit<
+  SP.AddPiecesSuccess,
+  'txStatus' | 'addMessageOk' | 'piecesAdded' | 'pieceCount' | 'confirmedPieceIds'
+>
+
+/**
+ * Custom type for JSON
+ * It will be used to store JSON data in the database
+ */
+export const jsonType = customType<{ data: unknown }>({
+  dataType() {
+    return 'text'
+  },
+  toDriver(value) {
+    return Json.stringify(value)
+  },
+  fromDriver(value) {
+    return Json.parse(value as string)
+  },
+})
+
+export const bigintType = customType<{ data: bigint }>({
+  dataType() {
+    return 'text'
+  },
+  toDriver(value) {
+    return value.toString()
+  },
+  fromDriver(value) {
+    return BigInt(value as string)
+  },
+})
+
+export type RepairInsert = typeof repairs.$inferInsert
+export type RepairSelect = typeof repairs.$inferSelect
+export type RepairUpdate = Partial<RepairInsert>
+
+export const repairs = table('repairs', {
+  id: t.int().primaryKey({ autoIncrement: true }),
+  status: t.text().$type<RepairStatus>().notNull().default('pending'),
+  repairProviderId: bigintType('repair_provider_id').notNull(),
+  targetProviderId: bigintType('target_provider_id').notNull(),
+  targetProviderUrl: t.text('target_provider_url').notNull(),
+  targetDataSetId: bigintType('target_data_set_id'),
+  blockNumber: bigintType('block_number').notNull(),
+  createdAt: t.integer('created_at').notNull(),
+  updatedAt: t.integer('updated_at').notNull(),
+})
+
+export type OperationInsert = typeof operations.$inferInsert
+export type OperationSelect = typeof operations.$inferSelect
+
+export const operations = table('operations', {
+  id: t.int().primaryKey({ autoIncrement: true }),
+  repairId: t
+    .int('repair_id')
+    .references((): AnySQLiteColumn => repairs.id)
+    .notNull(),
+  type: t.text().$type<OperationType>().notNull(),
+  status: t.text().$type<OperationStatus>().notNull().default('pending'),
+  cid: t.text().notNull(),
+  metadata: jsonType().$type<MetadataObject>().notNull(),
+  alternateProvider: t.text('alternate_provider').notNull(),
+  result: jsonType().$type<OperationResult>(),
+  error: t.text(),
+  createdAt: t.integer('created_at').notNull(),
+  updatedAt: t.integer('updated_at').notNull(),
+})
+
+export const repairRelations = relations(repairs, ({ many }) => ({
+  operations: many(operations),
+}))
+
+export const operationRelations = relations(operations, ({ one }) => ({
+  repair: one(repairs, {
+    fields: [operations.repairId],
+    references: [repairs.id],
+  }),
+}))

package/src/middleware.ts

@@ -0,0 +1,34 @@
+import type { Chain } from '@filoz/synapse-core/chains'
+import { drizzle as drizzlePostgres } from 'drizzle-orm/node-postgres'
+import { middleware, z } from 'incur'
+import type { Account, Client, Transport } from 'viem'
+import * as indexerSchema from './indexer-schema.ts'
+import type { IndexerDatabase, LocalDatabase } from './types.ts'
+import { config, createLocalDatabase, getClient } from './utils.ts'
+
+export const contextSchema = z.object({
+  indexerDb: z.custom<IndexerDatabase>(),
+  localDb: z.custom<LocalDatabase>(),
+  config: z.custom<typeof config>(),
+  client: z.custom<Client<Transport, Chain, Account>>(),
+  chain: z.custom<Chain>(),
+})
+
+export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
+  const { dbPath, chainId, indexerMainnetUrl, indexerCalibrationUrl } = config.store
+  const localDb = await createLocalDatabase(dbPath)
+  const indexerDb = drizzlePostgres(chainId === 314 ? indexerMainnetUrl : indexerCalibrationUrl, {
+    schema: indexerSchema,
+  })
+
+  const { client, chain } = getClient(chainId)
+  c.set('localDb', localDb)
+  c.set('indexerDb', indexerDb)
+  c.set('config', config)
+  c.set('client', client)
+  c.set('chain', chain)
+  await next()
+
+  localDb.$client.close()
+  await indexerDb.$client.end()
+})

package/src/pipeline/create-datasets.ts

@@ -0,0 +1,70 @@
+import * as p from '@clack/prompts'
+import * as SP from '@filoz/synapse-core/sp'
+import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
+import { getRepairDataset } from '../db/get-repair-dataset.ts'
+import { repairUpdate } from '../db/repair-update.ts'
+import type { RepairSelect } from '../local-schema.ts'
+import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import { getRepairDatasetMetadata, hashLink } from '../utils.ts'
+
+export type EnsureRepairDatasetOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  client: WalletClient
+  repair: RepairSelect
+}
+
+/**
+ * Ensure the repair dataset exists by creating it if it doesn't.
+ *
+ * @param options - The options for ensuring the repair dataset.
+ * @returns {Promise<bigint>} - The ID of the created dataset.
+ */
+export async function ensureRepairDataset({
+  localDb,
+  indexerDb,
+  client,
+  repair,
+}: EnsureRepairDatasetOptions): Promise<bigint> {
+  const log = p.taskLog({
+    title: 'Ensuring repair dataset',
+  })
+  const provider = await getPDPProvider(client, {
+    providerId: repair.targetProviderId,
+  })
+
+  if (!provider) throw new Error(`Target provider ${repair.targetProviderId} not found or inactive`)
+
+  let datasetId: bigint | null = null
+  // check if dataset already exists
+  const existingDatasetId = await getRepairDataset({
+    indexerDb,
+    providerId: repair.targetProviderId,
+    payer: client.account.address,
+  })
+
+  if (existingDatasetId) {
+    datasetId = existingDatasetId
+    log.success(`Data set #${datasetId} already exists at ${provider.pdp.serviceURL}`)
+  } else {
+    const { txHash, statusUrl } = await SP.createDataSet(client, {
+      payee: provider.payee,
+      serviceURL: provider.pdp.serviceURL,
+      payer: client.account.address,
+      cdn: false,
+      metadata: getRepairDatasetMetadata(),
+    })
+    log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
+    const waitForResult = await SP.waitForCreateDataSet({
+      statusUrl,
+    })
+    datasetId = waitForResult.dataSetId
+    log.success(`Data set #${datasetId} created at ${provider.pdp.serviceURL}`)
+  }
+  await repairUpdate({
+    localDb,
+    repairId: repair.id,
+    targetDataSetId: datasetId,
+  })
+  return datasetId
+}

package/src/pipeline/pull.ts

@@ -0,0 +1,255 @@
+import { taskLog } from '@clack/prompts'
+import * as Piece from '@filoz/synapse-core/piece'
+import { createPieceUrlPDP } from '@filoz/synapse-core/piece'
+import * as SP from '@filoz/synapse-core/sp'
+import { and, asc, eq, gt, inArray } from 'drizzle-orm'
+import PQueue from 'p-queue'
+import { getTargetDataset } from '../db/get-target-dataset.ts'
+import { syncPiecesOnchain } from '../db/sync-pieces-onchain.ts'
+import { updateOperation } from '../db/update-operation.ts'
+import { upsertOperations } from '../db/upsert-operations.ts'
+import type { OperationSelect, RepairSelect } from '../local-schema.ts'
+import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import { hashLink } from '../utils.ts'
+
+/** Pending `add_piece` operations batched for a single pull job. */
+export type PullPiecesBatch = {
+  operations: OperationSelect[]
+}
+
+export type RunPullPiecesPhaseOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  concurrency: number
+  batchSize: number
+  client: WalletClient
+  reset: boolean
+}
+
+/** Mock pull worker: logs each batch and its piece CIDs. */
+export function createPullPiecesWorker({
+  localDb,
+  indexerDb,
+  repair,
+  client,
+  state,
+  log,
+}: {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  client: WalletClient
+  state: {
+    totalBatches: number
+    totalOperations: number
+    completedOperations: number
+    failedOperations: number
+  }
+  log: ReturnType<typeof taskLog>
+}) {
+  return async (batch: PullPiecesBatch, batchNumber: number) => {
+    let completedCids = 0
+    let failedCids = 0
+    const cidToOperation = new Map<string, OperationSelect>()
+
+    const spin = log.group(`Batch ${batchNumber}/${state.totalBatches}`)
+    spin.message(`Pull 0 completed, 0 failed`)
+
+    try {
+      const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
+
+      for (const operation of batch.operations) {
+        cidToOperation.set(operation.cid, operation)
+      }
+
+      // sync pieces onchain to avoid duplicates
+      const completedOperations1 = await syncPiecesOnchain({
+        indexerDb,
+        localDb,
+        dataSetId: dataset.dataSetId,
+        cidToOperation,
+      })
+      state.completedOperations += completedOperations1
+      completedCids += completedOperations1
+
+      // create pull pieces
+      const pullPieces: SP.PullPieceInput[] = []
+      for (const [cid, operation] of cidToOperation) {
+        const pieceCid = Piece.from(cid)
+        const sourceUrl = createPieceUrlPDP({
+          cid,
+          serviceURL: operation.alternateProvider,
+        })
+        pullPieces.push({ pieceCid, sourceUrl })
+      }
+
+      if (pullPieces.length > 0) {
+        // wait for pull pieces
+        const pullResult = await SP.waitForPullPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: pullPieces,
+          timeout: 1000 * 60 * 30,
+          onStatus: (_status) => {
+            const completed = _status.pieces.filter((piece) => piece.status === 'complete').length
+            const failed = _status.pieces.filter((piece) => piece.status === 'failed').length
+            spin.message(`Pull ${completed} completed, ${failed} failed`)
+          },
+        })
+
+        for (const { pieceCid, status } of pullResult.pieces) {
+          const cid = pieceCid.toString()
+          const operation = cidToOperation.get(cid)
+          if (!operation) {
+            console.log(`operation not found for cid ${cid}`)
+            continue
+          }
+
+          if (status !== 'complete') {
+            state.failedOperations++
+            failedCids++
+            cidToOperation.delete(cid)
+            await updateOperation({
+              localDb,
+              operationId: operation.id,
+              status: 'failed',
+              error: `pull failed with status ${status}`,
+            })
+          }
+        }
+      }
+
+      // sync against indexer to avoid duplicates
+      const completedOperations2 = await syncPiecesOnchain({
+        indexerDb,
+        localDb,
+        dataSetId: dataset.dataSetId,
+        cidToOperation,
+      })
+      state.completedOperations += completedOperations2
+      completedCids += completedOperations2
+
+      const commitPieces: SP.addPieces.PieceType[] = []
+      for (const [cid] of cidToOperation) {
+        commitPieces.push({
+          pieceCid: Piece.from(cid),
+        })
+      }
+
+      if (commitPieces.length > 0) {
+        const addPiecesResult = await SP.addPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: commitPieces,
+        })
+
+        spin.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
+        const addPiecesResult2 = await SP.waitForAddPieces(addPiecesResult)
+        state.completedOperations += cidToOperation.size
+        completedCids += cidToOperation.size
+        await upsertOperations({
+          localDb,
+          operations: Array.from(cidToOperation.values()).map((operation) => ({
+            ...operation,
+            status: 'completed',
+            error: null,
+            result: { dataSetId: addPiecesResult2.dataSetId, txHash: addPiecesResult2.txHash },
+          })),
+        })
+      }
+      spin.success(`Batch ${batchNumber}/${state.totalBatches} ${completedCids} added, ${failedCids} failed`)
+    } catch (error) {
+      state.failedOperations += cidToOperation.size
+      const message = error instanceof Error ? error.message : 'Unknown error'
+      spin.error(`Batch ${batchNumber}/${state.totalBatches} - ${message.replace(/\n/g, ' ')}`)
+      await upsertOperations({
+        localDb,
+        operations: Array.from(cidToOperation.values()).map((operation) => ({
+          ...operation,
+          status: 'failed',
+          error: message,
+        })),
+      })
+    }
+  }
+}
+
+/**
+ * Pull pending `add_piece` operations without loading the whole repair into memory.
+ *
+ * Pending piece operations are fetched lazily and queued with bounded backpressure. Failed piece
+ * operations are intentionally skipped unless `reset` is set.
+ */
+export async function runPullPiecesPhase({
+  localDb,
+  indexerDb,
+  repair,
+  concurrency,
+  batchSize,
+  client,
+  reset,
+}: RunPullPiecesPhaseOptions): Promise<void> {
+  const localSchema = localDb._.fullSchema
+  const pullConcurrency = Math.max(1, concurrency)
+  const pullBatchSize = Math.max(1, batchSize)
+  let pullCursor = 0
+
+  const totalOperations = await localDb.$count(
+    localSchema.operations,
+    and(
+      eq(localSchema.operations.repairId, repair.id),
+      eq(localSchema.operations.type, 'add_piece'),
+      inArray(localSchema.operations.status, reset ? ['pending', 'failed'] : ['pending'])
+    )
+  )
+  let batchNumber = 0
+  const state = {
+    totalBatches: Math.ceil(totalOperations / pullBatchSize),
+    totalOperations,
+    completedOperations: 0,
+    failedOperations: 0,
+  }
+
+  const log = taskLog({
+    title: 'Pulling pieces',
+    limit: 1,
+  })
+
+  async function getNextPullBatch(): Promise<PullPiecesBatch | null> {
+    const operations = await localDb.query.operations.findMany({
+      where: and(
+        eq(localSchema.operations.repairId, repair.id),
+        eq(localSchema.operations.type, 'add_piece'),
+        inArray(localSchema.operations.status, reset ? ['pending', 'failed'] : ['pending']),
+        gt(localSchema.operations.id, pullCursor)
+      ),
+      orderBy: [asc(localSchema.operations.id)],
+      limit: pullBatchSize,
+    })
+    if (operations.length === 0) {
+      return null
+    }
+
+    pullCursor = operations.at(-1)?.id ?? pullCursor
+    return { operations }
+  }
+
+  const pullPiecesWorker = createPullPiecesWorker({ localDb, indexerDb, repair, client, state, log })
+  const pullPiecesQueue = new PQueue({ concurrency: pullConcurrency })
+
+  while (true) {
+    await pullPiecesQueue.onSizeLessThan(pullConcurrency)
+    const batch = await getNextPullBatch()
+    if (!batch) break
+    batchNumber++
+    const currentBatchNumber = batchNumber
+    pullPiecesQueue.add(() => pullPiecesWorker(batch, currentBatchNumber)).catch(console.error)
+  }
+
+  await pullPiecesQueue.onIdle()
+
+  log.success(`Pulled ${state.completedOperations} pieces, ${state.failedOperations} failed`)
+}

package/src/types.ts

@@ -0,0 +1,41 @@
+import type { Chain } from '@filoz/synapse-core/chains'
+import type { Client } from '@libsql/client'
+import type { LibSQLDatabase } from 'drizzle-orm/libsql'
+import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
+import type { z } from 'incur'
+import type { Pool } from 'pg'
+import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem'
+import type * as indexerSchema from './indexer-schema.ts'
+import type * as localSchema from './local-schema.ts'
+import type { contextSchema } from './middleware.ts'
+export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
+  $client: Client
+}
+
+export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
+  $client: Pool
+}
+
+export interface Config {
+  privateKey: Hex
+  indexerMainnetUrl: string
+  indexerCalibrationUrl: string
+  chainId: number
+  dbPath: string
+}
+
+export type WalletClient = ViemClient<Transport, Chain, Account>
+
+export type Context = z.infer<typeof contextSchema>
+
+/**
+ * Provider details used for repair selection and CID replica lookup.
+ */
+export type RepairProvider = {
+  providerId: bigint
+  providerAddress: Address
+  name: string
+  serviceUrl: string
+  approved: boolean
+  endorsed: boolean
+}