@filoz/repair-cli 0.3.0 → 0.4.0

package/src/db/second-copy-create.ts

@@ -0,0 +1,147 @@
+import { readFile } from 'node:fs/promises'
+import { taskLog } from '@clack/prompts'
+import { getBlockNumber } from 'viem/actions'
+import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
+import type { OperationInsert } from '../local-schema.ts'
+import type { Context } from '../types.ts'
+import { getRepairProvider } from './get-repair-provider.ts'
+
+const OPERATIONS_INSERT_BATCH_SIZE = 500
+
+export interface SecondCopyCreateOptions extends Context {
+  piecesFile: string
+  sourceProviderUrl: string
+  targetProviderId: bigint
+}
+
+type PiecesFile = {
+  aggregates: Array<{ piece_cid?: unknown }>
+}
+
+/**
+ * Normalize a provider service URL for storage and equality checks.
+ */
+export function normalizeProviderUrl(url: string): string {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    throw new RepairCreationError(`Invalid source provider URL: ${url}`)
+  }
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new RepairCreationError(`Source provider URL must be http(s): ${url}`)
+  }
+
+  // Drop trailing slash so endpoint comparisons are stable.
+  return parsed.toString().replace(/\/$/, '')
+}
+
+/**
+ * Parse a second-copy pieces file and return unique aggregate piece CIDs.
+ */
+export async function parsePieceCidsFromFile(piecesFile: string): Promise<string[]> {
+  let raw: unknown
+  try {
+    raw = JSON.parse(await readFile(piecesFile, 'utf8'))
+  } catch (error) {
+    throw new RepairCreationError(
+      error instanceof Error ? `Failed to read pieces file: ${error.message}` : 'Failed to read pieces file'
+    )
+  }
+
+  if (raw == null || typeof raw !== 'object' || !Array.isArray((raw as PiecesFile).aggregates)) {
+    throw new RepairCreationError('Pieces file must contain an "aggregates" array')
+  }
+
+  const cids = new Set<string>()
+  for (const aggregate of (raw as PiecesFile).aggregates) {
+    if (typeof aggregate?.piece_cid !== 'string' || aggregate.piece_cid.length === 0) {
+      throw new RepairCreationError('Each aggregate must include a non-empty "piece_cid" string')
+    }
+    cids.add(aggregate.piece_cid)
+  }
+
+  if (cids.size === 0) {
+    throw new RepairCreationError('Pieces file contains no piece CIDs')
+  }
+
+  return [...cids]
+}
+
+/**
+ * Prepare a second-copy job from a pieces file and an explicit source provider URL.
+ *
+ * @param options - The options for creating a second-copy job.
+ * @returns The ID of the created local job.
+ */
+export async function secondCopyCreate(options: SecondCopyCreateOptions): Promise<number> {
+  const { indexerDb, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
+  const localSchema = localDb._.fullSchema
+  const now = Date.now()
+  const blockNumber = await getBlockNumber(client)
+  const normalizedSourceUrl = normalizeProviderUrl(sourceProviderUrl)
+
+  const log = taskLog({
+    title: 'Creating second-copy',
+    limit: 10,
+    retainLog: true,
+  })
+
+  const targetProvider = await getRepairProvider({
+    indexerDb,
+    providerId: targetProviderId,
+  })
+
+  if (!targetProvider) {
+    throw new NoAlternateProviderError(targetProviderId)
+  }
+
+  const normalizedTargetUrl = normalizeProviderUrl(targetProvider.serviceUrl)
+  if (normalizedSourceUrl === normalizedTargetUrl) {
+    throw new RepairCreationError('Source provider URL must differ from the target provider URL')
+  }
+
+  const pieceCids = await parsePieceCidsFromFile(piecesFile)
+  log.message(`Loaded ${pieceCids.length} unique piece CIDs from ${piecesFile}`)
+
+  const [repair] = await localDb
+    .insert(localSchema.repairs)
+    .values({
+      kind: 'second_copy',
+      repairProviderId: 0n,
+      repairDataSetId: null,
+      sourceProviderUrl: normalizedSourceUrl,
+      targetProviderId: targetProvider.providerId,
+      targetProviderUrl: targetProvider.serviceUrl,
+      targetDataSetId: null,
+      blockNumber,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .returning({ id: localSchema.repairs.id })
+
+  if (!repair) throw new RepairCreationError('Failed to create second-copy row')
+
+  let totalOperations = 0
+  for (let offset = 0; offset < pieceCids.length; offset += OPERATIONS_INSERT_BATCH_SIZE) {
+    const batchCids = pieceCids.slice(offset, offset + OPERATIONS_INSERT_BATCH_SIZE)
+    const operations: OperationInsert[] = batchCids.map((cid) => ({
+      repairId: repair.id,
+      type: 'add_piece',
+      status: 'pending',
+      cid,
+      metadata: {},
+      alternateProvider: normalizedSourceUrl,
+      createdAt: now,
+      updatedAt: now,
+    }))
+
+    await localDb.insert(localSchema.operations).values(operations)
+    totalOperations += operations.length
+    log.message(`Inserted ${operations.length} operations (${totalOperations}/${pieceCids.length})`)
+  }
+
+  log.success(`Created second-copy ${repair.id} with ${totalOperations} operations`, { showLog: true })
+  return repair.id
+}

package/src/local-schema.ts

@@ -6,6 +6,7 @@ import { customType, sqliteTable as table } from 'drizzle-orm/sqlite-core'
 import * as Json from 'iso-base/json'
 
 export type RepairStatus = 'pending' | 'completed' | 'failed'
+export type RepairKind = 'repair' | 'replicate' | 'second_copy'
 export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped'
 export type OperationType = 'create_dataset' | 'add_piece'
 
@@ -44,8 +45,10 @@ export type RepairUpdate = Partial<RepairInsert>
 export const repairs = table('repairs', {
   id: t.int().primaryKey({ autoIncrement: true }),
   status: t.text().$type<RepairStatus>().notNull().default('pending'),
+  kind: t.text().$type<RepairKind>().notNull().default('repair'),
   repairProviderId: bigintType('repair_provider_id').notNull(),
   repairDataSetId: bigintType('repair_data_set_id'),
+  sourceProviderUrl: t.text('source_provider_url'),
   targetProviderId: bigintType('target_provider_id').notNull(),
   targetProviderUrl: t.text('target_provider_url').notNull(),
   targetDataSetId: bigintType('target_data_set_id'),

package/src/pipeline/add-pieces.ts

@@ -75,7 +75,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
           dataSetId: dataset.dataSetId,
           clientDataSetId: dataset.clientDataSetId,
           pieces: operationsToPullPieces(operations),
-          timeout: 1000 * 60 * 30,
+          timeout: 1000 * 60 * 30, // 30 minutes
           onStatus: (status) => {
             const completed = status.pieces.filter((piece) => piece.status === 'complete').length
             const failed = status.pieces.filter((piece) => piece.status === 'failed').length

package/src/utils.ts

@@ -81,6 +81,7 @@ export async function createLocalDatabase(dbPath: string): Promise<LocalDatabase
   }) as LocalDatabase
 
   await localDb.$client.execute('PRAGMA journal_mode = WAL')
+  await migrateLocalDatabase(localDb)
 
   return localDb
 }
@@ -109,6 +110,25 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
   } catch {
     // Column already exists on databases created with the updated schema.
   }
+  try {
+    await db.$client.execute(`
+      ALTER TABLE repairs ADD COLUMN kind text NOT NULL DEFAULT 'repair'
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
+  try {
+    await db.$client.execute(`
+      ALTER TABLE repairs ADD COLUMN source_provider_url text
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
+  await db.$client.execute(`
+    UPDATE repairs
+    SET kind = 'replicate'
+    WHERE repair_data_set_id IS NOT NULL AND (kind IS NULL OR kind = 'repair')
+  `)
   await db.$client.execute(`
     CREATE TABLE IF NOT EXISTS operations (
       id integer PRIMARY KEY AUTOINCREMENT NOT NULL,