@filoz/repair-cli 0.2.1 → 0.4.0

package/src/db/update-operation.ts

@@ -6,19 +6,19 @@ export type UpdateOperationOptions = {
   localDb: LocalDatabase
   operationId: number
   status: localSchema.OperationStatus
-  result?: localSchema.OperationResult | null
+  txHash?: string | null
   error?: string | null
 }
 
 /**
  * Updates an operation in the database.
  */
-export async function updateOperation({ localDb, operationId, status, result, error }: UpdateOperationOptions) {
+export async function updateOperation({ localDb, operationId, status, txHash, error }: UpdateOperationOptions) {
   await localDb
     .update(localSchema.operations)
     .set({
       status,
-      result,
+      txHash,
       error: error ?? null,
       updatedAt: Date.now(),
     })

package/src/db/upsert-operations.ts

@@ -18,6 +18,6 @@ export async function upsertOperations({ localDb, operations }: UpsertOperations
     .values(operations.map((operation) => ({ ...operation, updatedAt: now })))
     .onConflictDoUpdate({
       target: localDb._.fullSchema.operations.id,
-      set: buildConflictUpdateColumns(localSchema.operations, ['status', 'error', 'updatedAt', 'result']),
+      set: buildConflictUpdateColumns(localSchema.operations, ['status', 'error', 'updatedAt', 'txHash']),
     })
 }

package/src/local-schema.ts

@@ -1,5 +1,4 @@
 import type { MetadataObject } from '@filoz/synapse-core'
-import type * as SP from '@filoz/synapse-core/sp'
 import { relations } from 'drizzle-orm'
 import type { AnySQLiteColumn } from 'drizzle-orm/sqlite-core'
 import * as t from 'drizzle-orm/sqlite-core'
@@ -7,14 +6,10 @@ import { customType, sqliteTable as table } from 'drizzle-orm/sqlite-core'
 import * as Json from 'iso-base/json'
 
 export type RepairStatus = 'pending' | 'completed' | 'failed'
+export type RepairKind = 'repair' | 'replicate' | 'second_copy'
 export type OperationStatus = 'pending' | 'completed' | 'failed' | 'skipped'
 export type OperationType = 'create_dataset' | 'add_piece'
 
-export type OperationResult = Omit<
-  SP.AddPiecesSuccess,
-  'txStatus' | 'addMessageOk' | 'piecesAdded' | 'pieceCount' | 'confirmedPieceIds'
->
-
 /**
  * Custom type for JSON
  * It will be used to store JSON data in the database
@@ -50,8 +45,10 @@ export type RepairUpdate = Partial<RepairInsert>
 export const repairs = table('repairs', {
   id: t.int().primaryKey({ autoIncrement: true }),
   status: t.text().$type<RepairStatus>().notNull().default('pending'),
+  kind: t.text().$type<RepairKind>().notNull().default('repair'),
   repairProviderId: bigintType('repair_provider_id').notNull(),
   repairDataSetId: bigintType('repair_data_set_id'),
+  sourceProviderUrl: t.text('source_provider_url'),
   targetProviderId: bigintType('target_provider_id').notNull(),
   targetProviderUrl: t.text('target_provider_url').notNull(),
   targetDataSetId: bigintType('target_data_set_id'),
@@ -74,7 +71,7 @@ export const operations = table('operations', {
   cid: t.text().notNull(),
   metadata: jsonType().$type<MetadataObject>().notNull(),
   alternateProvider: t.text('alternate_provider').notNull(),
-  result: jsonType().$type<OperationResult>(),
+  txHash: t.text('tx_hash'),
   error: t.text(),
   createdAt: t.integer('created_at').notNull(),
   updatedAt: t.integer('updated_at').notNull(),

package/src/middleware.ts

@@ -13,6 +13,7 @@ export const contextSchema = z.object({
   client: z.custom<Client<Transport, Chain, Account>>(),
   chain: z.custom<Chain>(),
   source: z.string(),
+  isInteractive: z.boolean(),
 })
 
 export const contextMiddleware = middleware<typeof contextSchema>(async (c, next) => {
@@ -38,6 +39,7 @@ export const contextMiddleware = middleware<typeof contextSchema>(async (c, next
   c.set('client', client)
   c.set('chain', chain)
   c.set('source', source)
+  c.set('isInteractive', !c.agent && !c.formatExplicit)
   await next()
 
   localDb.$client.close()

package/src/pipeline/add-pieces.ts

@@ -9,7 +9,7 @@ import { repairUpdate } from '../db/repair-update.ts'
 import { upsertOperations } from '../db/upsert-operations.ts'
 import type { OperationSelect, RepairSelect } from '../local-schema.ts'
 import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
-import { excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
+import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
 
 export type RunPullPiecesPhaseOptions = {
   localDb: LocalDatabase
@@ -58,6 +58,15 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
       if (isRepair) {
         operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
       }
+      // A previous run may have submitted a transaction and crashed before marking rows completed.
+      const operationsBeforeConfirmationCheck = operations.length
+      operations = await completeConfirmedOperations({ localDb, client, operations })
+      const confirmedOperations = operationsBeforeConfirmationCheck - operations.length
+      if (confirmedOperations > 0) {
+        state.completedOperations += confirmedOperations
+        completedOps += confirmedOperations
+      }
+
       group.message(`Pulling ${operations.length} pieces...`)
       // pull pieces
       if (operations.length > 0) {
@@ -66,7 +75,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
           dataSetId: dataset.dataSetId,
           clientDataSetId: dataset.clientDataSetId,
           pieces: operationsToPullPieces(operations),
-          timeout: 1000 * 60 * 30,
+          timeout: 1000 * 60 * 30, // 30 minutes
           onStatus: (status) => {
             const completed = status.pieces.filter((piece) => piece.status === 'complete').length
             const failed = status.pieces.filter((piece) => piece.status === 'failed').length
@@ -105,9 +114,16 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
             metadata: isRepair ? undefined : operation.metadata,
           })),
         })
+        await upsertOperations({
+          localDb,
+          operations: operations.map((operation) => ({
+            ...operation,
+            txHash: addPiecesResult.txHash,
+          })),
+        })
 
         group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
-        const addPiecesResult2 = await SP.waitForAddPieces(addPiecesResult)
+        const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
         state.completedOperations += operations.length
         completedOps += operations.length
         await upsertOperations({
@@ -116,7 +132,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
             ...operation,
             status: 'completed',
             error: null,
-            result: { dataSetId: addPiecesResult2.dataSetId, txHash: addPiecesResult2.txHash },
+            txHash: waitForAddPieces.txHash,
           })),
         })
       }

package/src/pipeline/create-datasets.ts

@@ -2,6 +2,7 @@ import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
 import { eq } from 'drizzle-orm'
+import type { Address } from 'viem'
 import { findRepairDataset } from '../db/find-repair-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import type { RepairSelect } from '../local-schema.ts'
@@ -14,6 +15,7 @@ export type EnsureRepairDatasetOptions = {
   indexerDb: IndexerDatabase
   client: WalletClient
   repair: RepairSelect
+  payer: Address
 }
 
 /**
@@ -21,7 +23,14 @@ export type EnsureRepairDatasetOptions = {
  *
  * @param options - The options for ensuring the repair dataset.
  */
-export async function ensureRepairDataset({ source, localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
+export async function ensureRepairDataset({
+  source,
+  localDb,
+  indexerDb,
+  client,
+  repair,
+  payer,
+}: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring repair dataset',
   })
@@ -36,7 +45,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
   const existingDatasetId = await findRepairDataset({
     indexerDb,
     providerId: repair.targetProviderId,
-    payer: client.account.address,
+    payer,
     source,
   })
 
@@ -47,7 +56,7 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
     const { txHash, statusUrl } = await SP.createDataSet(client, {
       payee: provider.payee,
       serviceURL: provider.pdp.serviceURL,
-      payer: client.account.address,
+      payer,
       cdn: false,
       metadata: {
         source,
@@ -74,7 +83,13 @@ export async function ensureRepairDataset({ source, localDb, indexerDb, client,
  *
  * @param options - The options for ensuring the replication dataset.
  */
-export async function ensureReplicateDataset({ localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
+export async function ensureReplicateDataset({
+  localDb,
+  indexerDb,
+  client,
+  repair,
+  payer,
+}: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring replication dataset',
   })
@@ -110,7 +125,7 @@ export async function ensureReplicateDataset({ localDb, indexerDb, client, repai
   const { txHash, statusUrl } = await SP.createDataSet(client, {
     payee: provider.payee,
     serviceURL: provider.pdp.serviceURL,
-    payer: client.account.address,
+    payer,
     cdn: sourceDataSet.withCdn,
     metadata: sourceDataSet.metadata ?? undefined,
   })

package/src/utils.ts

@@ -1,8 +1,7 @@
-import type { MetadataObject } from '@filoz/synapse-core'
 import { type Chain, getChain } from '@filoz/synapse-core/chains'
 import * as Piece from '@filoz/synapse-core/piece'
 import type * as SP from '@filoz/synapse-core/sp'
-import { getTableColumns, type SQL, sql } from 'drizzle-orm'
+import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
 import type { PgTable } from 'drizzle-orm/pg-core'
 import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
@@ -11,12 +10,13 @@ import { Conf } from 'iso-conf'
 import { request } from 'iso-web/http'
 import pLocate from 'p-locate'
 import terminalLink from 'terminal-link'
-import { createWalletClient, type Hex, http } from 'viem'
+import { createWalletClient, type Hash, type Hex, http, TransactionReceiptNotFoundError } from 'viem'
 import { privateKeyToAccount } from 'viem/accounts'
+import { getTransactionReceipt } from 'viem/actions'
 import packageJson from '../package.json' with { type: 'json' }
 import type { OperationSelect } from './local-schema.ts'
 import * as schema from './local-schema.ts'
-import type { LocalDatabase } from './types.ts'
+import type { LocalDatabase, WalletClient } from './types.ts'
 
 export const configSchema = z.object({
   privateKey: z.string().optional(),
@@ -81,6 +81,7 @@ export async function createLocalDatabase(dbPath: string): Promise<LocalDatabase
   }) as LocalDatabase
 
   await localDb.$client.execute('PRAGMA journal_mode = WAL')
+  await migrateLocalDatabase(localDb)
 
   return localDb
 }
@@ -109,6 +110,25 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
   } catch {
     // Column already exists on databases created with the updated schema.
   }
+  try {
+    await db.$client.execute(`
+      ALTER TABLE repairs ADD COLUMN kind text NOT NULL DEFAULT 'repair'
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
+  try {
+    await db.$client.execute(`
+      ALTER TABLE repairs ADD COLUMN source_provider_url text
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
+  await db.$client.execute(`
+    UPDATE repairs
+    SET kind = 'replicate'
+    WHERE repair_data_set_id IS NOT NULL AND (kind IS NULL OR kind = 'repair')
+  `)
   await db.$client.execute(`
     CREATE TABLE IF NOT EXISTS operations (
       id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
@@ -118,13 +138,27 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
       cid text NOT NULL,
       metadata text NOT NULL,
       alternate_provider text NOT NULL,
-      result text,
+      tx_hash text,
       error text,
       created_at integer NOT NULL,
       updated_at integer NOT NULL,
       FOREIGN KEY (repair_id) REFERENCES repairs(id)
     )
   `)
+  try {
+    await db.$client.execute(`
+      ALTER TABLE operations ADD COLUMN tx_hash text
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
+  try {
+    await db.$client.execute(`
+      ALTER TABLE operations DROP COLUMN result
+    `)
+  } catch {
+    // Column does not exist on databases created with the updated schema.
+  }
 }
 
 /**
@@ -139,6 +173,62 @@ export function hashLink(hash: string, chain: Chain) {
   return link
 }
 
+/**
+ * Mark operations with successful mined transactions as completed.
+ *
+ * @returns Operations that are not completed yet.
+ */
+export async function completeConfirmedOperations({
+  localDb,
+  client,
+  operations,
+}: {
+  localDb: LocalDatabase
+  client: WalletClient
+  operations: OperationSelect[]
+}): Promise<OperationSelect[]> {
+  const hashes = new Set(
+    operations.map((operation) => operation.txHash).filter((txHash): txHash is Hash => txHash != null)
+  )
+  const completedHashes = new Set<Hash>()
+
+  for (const hash of hashes) {
+    try {
+      const receipt = await getTransactionReceipt(client, { hash })
+      if (receipt.status === 'success') {
+        completedHashes.add(hash)
+      }
+    } catch (error) {
+      if (!(error instanceof TransactionReceiptNotFoundError)) {
+        throw error
+      }
+    }
+  }
+
+  const completedOperations = operations.filter(
+    (operation) => operation.txHash != null && completedHashes.has(operation.txHash as Hash)
+  )
+
+  if (completedOperations.length > 0) {
+    await localDb
+      .update(schema.operations)
+      .set({
+        status: 'completed',
+        error: null,
+        updatedAt: Date.now(),
+      })
+      .where(
+        inArray(
+          schema.operations.id,
+          completedOperations.map((operation) => operation.id)
+        )
+      )
+  }
+  const completedOperationIds = new Set(completedOperations.map((operation) => operation.id))
+
+  return operations.filter((operation) => operation.status !== 'completed' && !completedOperationIds.has(operation.id))
+}
+
 /**
  * Get a piece from a service URL
  */