@filoz/repair-cli 0.2.1 → 0.4.0

package/src/commands/second-copy.ts

@@ -0,0 +1,187 @@
+import { and, desc, eq, inArray } from 'drizzle-orm'
+import { Cli, z } from 'incur'
+import { isAddress } from 'viem'
+import { repairDelete } from '../db/repair-delete.ts'
+import { secondCopyCreate } from '../db/second-copy-create.ts'
+import { contextMiddleware, contextSchema } from '../middleware.ts'
+import { runAddPieces } from '../pipeline/add-pieces.ts'
+import { ensureRepairDataset } from '../pipeline/create-datasets.ts'
+import { globalOptions } from '../utils.ts'
+
+export const secondCopy = Cli.create('second-copy', {
+  description: 'Second-copy piece commands',
+  vars: contextSchema,
+})
+
+secondCopy.command('create', {
+  description: 'Create a second-copy job from a pieces file',
+  options: globalOptions.extend({
+    piecesFile: z.string().describe('Path to JSON file with aggregates[].piece_cid'),
+    sourceProviderUrl: z.string().url().describe('Source provider URL to pull pieces from'),
+    targetProviderId: z.coerce.bigint().describe('Target provider ID for second-copy'),
+  }),
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const { piecesFile, sourceProviderUrl, targetProviderId } = c.options
+
+      const secondCopyId = await secondCopyCreate({
+        ...c.var,
+        piecesFile,
+        sourceProviderUrl,
+        targetProviderId,
+      })
+
+      return c.ok({
+        secondCopyId,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to create the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('list', {
+  description: 'List all second-copy jobs',
+  options: globalOptions,
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const localSchema = c.var.localDb._.fullSchema
+      const jobs = await c.var.localDb.query.repairs.findMany({
+        where: eq(localSchema.repairs.kind, 'second_copy'),
+        orderBy: [desc(localSchema.repairs.createdAt)],
+        with: {
+          operations: true,
+        },
+      })
+
+      const secondCopies = jobs.map((repair) => {
+        const { operations, ...job } = repair
+        return {
+          id: job.id,
+          status: job.status,
+          sourceProviderUrl: job.sourceProviderUrl,
+          targetProviderId: job.targetProviderId,
+          targetProviderUrl: job.targetProviderUrl,
+          targetDataSetId: job.targetDataSetId,
+          blockNumber: job.blockNumber,
+          operations: operations.length,
+          pending: operations.filter((operation) => operation.status === 'pending').length,
+          failed: operations.filter((operation) => operation.status === 'failed').length,
+          completed: operations.filter((operation) => operation.status === 'completed').length,
+          skipped: operations.filter((operation) => operation.status === 'skipped').length,
+        }
+      })
+
+      return c.ok({
+        secondCopies,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to list second-copy jobs',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('delete', {
+  description: 'Delete a second-copy job',
+  args: z.object({
+    secondCopyId: z.coerce.number().describe('Second-copy ID to delete'),
+  }),
+  options: globalOptions,
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const { deleted, operationsDeleted } = await repairDelete({
+        localDb: c.var.localDb,
+        repairId: c.args.secondCopyId,
+      })
+
+      if (!deleted) {
+        return c.error({
+          code: 'SECOND_COPY_NOT_FOUND',
+          message: 'Second-copy job not found',
+          retryable: false,
+        })
+      }
+
+      return c.ok({
+        secondCopyId: c.args.secondCopyId,
+        operationsDeleted,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to delete the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('run', {
+  description: 'Run a second-copy job',
+  args: z.object({
+    secondCopyId: z.coerce.number().describe('Second-copy ID to run'),
+  }),
+  options: globalOptions.extend({
+    concurrency: z.coerce.number().min(1).max(10).default(4).describe('Concurrency level'),
+    batchSize: z.coerce.number().min(1).max(40).default(40).describe('Max pieces per batch'),
+    payer: z.string().refine(isAddress, 'Invalid address').optional().describe('Payer address'),
+  }),
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const schema = c.var.localDb._.fullSchema
+      const payer = c.options.payer ?? c.var.client.account.address
+      const repair = await c.var.localDb.query.repairs.findFirst({
+        where: and(
+          eq(schema.repairs.id, c.args.secondCopyId),
+          inArray(schema.repairs.status, ['pending', 'failed']),
+          eq(schema.repairs.kind, 'second_copy')
+        ),
+      })
+      if (!repair) {
+        return c.error({
+          code: 'SECOND_COPY_NOT_FOUND',
+          message: 'Second-copy job not found, it may have already been run or completed',
+          retryable: false,
+        })
+      }
+
+      await ensureRepairDataset({
+        ...c.var,
+        repair,
+        payer,
+      })
+
+      await runAddPieces({
+        ...c.var,
+        repair,
+        concurrency: c.options.concurrency,
+        batchSize: c.options.batchSize,
+      })
+      return c.ok({
+        secondCopyId: repair.id,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to run the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})

package/src/commands/session-key.ts

@@ -0,0 +1,121 @@
+import * as p from '@clack/prompts'
+import * as SessionKey from '@filoz/synapse-core/session-key'
+import { Cli, z } from 'incur'
+import { type Address, isAddress } from 'viem'
+import { contextMiddleware, contextSchema } from '../middleware.ts'
+import { globalOptions, hashLink } from '../utils.ts'
+
+const sessionKeyAddressArgs = z.object({
+  address: z.string().refine(isAddress, 'Invalid session key address').describe('Session key address'),
+})
+
+export const sessionKey = Cli.create('session-key', {
+  description: 'Session key commands',
+  vars: contextSchema,
+})
+
+sessionKey.command('approve', {
+  description: 'Approve a session key for storage operations',
+  args: sessionKeyAddressArgs,
+  options: globalOptions.extend({
+    expiresInDays: z.coerce.number().gt(0).default(100).describe('Session key expiry duration in days'),
+    origin: z.string().optional().describe('Origin recorded on-chain for the authorization'),
+  }),
+  middleware: [contextMiddleware],
+  outputPolicy: 'agent-only',
+  run: async (c) => {
+    const { client, chain, source, isInteractive } = c.var
+    const address = c.args.address as Address
+    const expiresAt = BigInt(Math.floor(Date.now() / 1000 + c.options.expiresInDays * 24 * 60 * 60))
+    const spinner = p.spinner()
+
+    try {
+      if (isInteractive) {
+        spinner.start(`Approving session key ${address}...`)
+      }
+      const result = await SessionKey.loginSync(client, {
+        address,
+        expiresAt,
+        origin: c.options.origin ?? source,
+        onHash: (hash) => {
+          if (isInteractive) {
+            spinner.message(`Waiting for tx ${hashLink(hash, chain)} to be mined...`)
+          }
+        },
+      })
+
+      if (isInteractive) {
+        spinner.stop(`Session key ${address} approved successfully.`)
+      }
+
+      return c.ok({
+        address,
+        expiresAt: expiresAt.toString(),
+        transactionHash: result.receipt.transactionHash,
+      })
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : 'Failed to approve session key'
+      if (isInteractive) {
+        spinner.error(msg)
+      }
+      if (c.options.debug) {
+        console.error(error)
+      }
+      return c.error({
+        code: 'FAILED_TO_APPROVE_SESSION_KEY',
+        message: msg,
+      })
+    }
+  },
+})
+
+sessionKey.command('revoke', {
+  description: 'Revoke a session key for storage operations',
+  args: sessionKeyAddressArgs,
+  options: globalOptions.extend({
+    origin: z.string().optional().describe('Origin recorded on-chain for the revocation'),
+  }),
+  middleware: [contextMiddleware],
+  outputPolicy: 'agent-only',
+  run: async (c) => {
+    const { client, chain, source, isInteractive } = c.var
+    const address = c.args.address as Address
+    const spinner = p.spinner()
+
+    try {
+      if (isInteractive) {
+        spinner.start(`Revoking session key ${address}...`)
+      }
+      const result = await SessionKey.revokeSync(client, {
+        address,
+        origin: c.options.origin ?? source,
+        onHash: (hash) => {
+          if (isInteractive) {
+            spinner.message(`Waiting for tx ${hashLink(hash, chain)} to be mined...`)
+          }
+        },
+      })
+
+      if (isInteractive) {
+        spinner.stop(`Session key ${address} revoked successfully.`)
+      }
+
+      return c.ok({
+        address,
+        transactionHash: result.receipt.transactionHash,
+      })
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : 'Failed to revoke session key'
+      if (isInteractive) {
+        spinner.error(msg)
+      }
+      if (c.options.debug) {
+        console.error(error)
+      }
+      return c.error({
+        code: 'FAILED_TO_REVOKE_SESSION_KEY',
+        message: msg,
+      })
+    }
+  },
+})

package/src/commands/wallet.ts

@@ -1,9 +1,11 @@
 /** biome-ignore-all lint/suspicious/noConsole: cli */
+import * as p from '@clack/prompts'
 import { calibration } from '@filoz/synapse-core/chains'
 import * as ERC20 from '@filoz/synapse-core/erc20'
 import * as Pay from '@filoz/synapse-core/pay'
 import { claimTokens, formatBalance, formatFraction, parseUnits } from '@filoz/synapse-core/utils'
 import { Cli, z } from 'incur'
+import { isAddress } from 'viem'
 import { getBalance, waitForTransactionReceipt } from 'viem/actions'
 import { contextMiddleware, contextSchema } from '../middleware.ts'
 import { globalOptions, hashLink } from '../utils.ts'
@@ -17,8 +19,9 @@ wallet.command('fund', {
   description: 'Fund a calibration wallet from a faucet',
   options: globalOptions,
   middleware: [contextMiddleware],
-  async *run(c) {
-    const { client, chain } = c.var
+  outputPolicy: 'agent-only',
+  run: async (c) => {
+    const { client, chain, isInteractive } = c.var
 
     if (chain.id !== calibration.id) {
       return c.error({
@@ -27,28 +30,41 @@ wallet.command('fund', {
       })
     }
 
-    yield 'Funding wallet...'
+    const spinner = p.spinner()
     try {
+      if (isInteractive) {
+        spinner.start('Funding wallet...')
+      }
       const hashes = await claimTokens({ address: client.account.address })
 
-      yield `Waiting for tx ${hashLink(hashes[0].tx_hash, chain)} to be mined...`
+      if (isInteractive) {
+        spinner.message(`Waiting for tx ${hashLink(hashes[0].tx_hash, chain)} to be mined...`)
+      }
       await waitForTransactionReceipt(client, {
         hash: hashes[0].tx_hash,
       })
       const balance = await getBalance(client, {
         address: client.account.address,
       })
-      yield {
+      if (isInteractive) {
+        spinner.stop('Wallet funded successfully.')
+      }
+      return c.ok({
         address: client.account.address,
         balance: formatBalance({ value: balance }),
-      }
+        transactionHash: hashes[0].tx_hash,
+      })
     } catch (error) {
+      const msg = error instanceof Error ? error.message : 'Failed to fund wallet'
+      if (isInteractive) {
+        spinner.error(msg)
+      }
       if (c.options.debug) {
         console.error(error)
       }
       return c.error({
         code: 'FAILED_TO_FUND_WALLET',
-        message: 'Failed to fund wallet',
+        message: msg,
       })
     }
   },
@@ -56,23 +72,26 @@ wallet.command('fund', {
 
 wallet.command('balance', {
   description: 'Get wallet and pay account summary',
-  options: globalOptions,
+  options: globalOptions.extend({
+    address: z.string().refine(isAddress, 'Invalid address').optional().describe('Address to get balance for'),
+  }),
   middleware: [contextMiddleware],
   async run(c) {
     const { client } = c.var
+    const address = c.options.address ?? client.account.address
     const balanceFIL = await getBalance(client, {
-      address: client.account.address,
+      address,
     })
 
     const balanceUSDFC = await ERC20.balance(client, {
-      address: client.account.address,
+      address,
     })
 
     const summary = await Pay.getAccountSummary(client, {
-      address: client.account.address,
+      address,
     })
     return {
-      address: client.account.address,
+      address,
       fil: formatBalance({ value: balanceFIL }),
       usdfc: formatBalance({ value: balanceUSDFC.value }),
       pay: {
@@ -99,27 +118,42 @@ wallet.command('deposit', {
   }),
   options: globalOptions,
   middleware: [contextMiddleware],
-  async *run(c) {
-    const { client, chain } = c.var
+  outputPolicy: 'agent-only',
+  run: async (c) => {
+    const { client, chain, isInteractive } = c.var
+    const spinner = p.spinner()
 
     try {
-      yield `Depositing ${c.args.amount} tokens to wallet...`
+      if (isInteractive) {
+        spinner.start(`Depositing ${c.args.amount} USDFC to pay account...`)
+      }
       const hash = await Pay.depositAndApprove(client, {
         amount: parseUnits(c.args.amount),
       })
-      yield `Waiting for tx ${hashLink(hash, chain)} to be mined...`
+      if (isInteractive) {
+        spinner.message(`Waiting for tx ${hashLink(hash, chain)} to be mined...`)
+      }
       await waitForTransactionReceipt(client, {
         hash,
       })
-      yield `Deposit successful`
-      return
+      if (isInteractive) {
+        spinner.stop('Deposit successful.')
+      }
+      return c.ok({
+        amount: c.args.amount,
+        transactionHash: hash,
+      })
     } catch (error) {
+      const msg = error instanceof Error ? error.message : 'Failed to deposit'
+      if (isInteractive) {
+        spinner.error(msg)
+      }
       if (c.options.debug) {
         console.error(error)
       }
       return c.error({
         code: 'FAILED_TO_DEPOSIT',
-        message: (error as Error).message,
+        message: msg,
       })
     }
   },
@@ -132,27 +166,42 @@ wallet.command('withdraw', {
   }),
   options: globalOptions,
   middleware: [contextMiddleware],
-  async *run(c) {
-    const { client, chain } = c.var
+  outputPolicy: 'agent-only',
+  run: async (c) => {
+    const { client, chain, isInteractive } = c.var
+    const spinner = p.spinner()
 
     try {
-      yield `Withdrawing ${c.args.amount} USDFC from pay account...`
+      if (isInteractive) {
+        spinner.start(`Withdrawing ${c.args.amount} USDFC from pay account...`)
+      }
       const hash = await Pay.withdraw(client, {
         amount: parseUnits(c.args.amount),
       })
-      yield `Waiting for tx ${hashLink(hash, chain)} to be mined...`
+      if (isInteractive) {
+        spinner.message(`Waiting for tx ${hashLink(hash, chain)} to be mined...`)
+      }
       await waitForTransactionReceipt(client, {
         hash,
       })
-      yield `Withdrawal successful`
-      return
+      if (isInteractive) {
+        spinner.stop('Withdrawal successful.')
+      }
+      return c.ok({
+        amount: c.args.amount,
+        transactionHash: hash,
+      })
     } catch (error) {
+      const msg = error instanceof Error ? error.message : 'Failed to withdraw'
+      if (isInteractive) {
+        spinner.error(msg)
+      }
       if (c.options.debug) {
         console.error(error)
       }
       return c.error({
         code: 'FAILED_TO_WITHDRAW',
-        message: (error as Error).message,
+        message: msg,
       })
     }
   },

package/src/db/repair-create.ts

@@ -46,8 +46,10 @@ export async function repairCreate(options: RepairCreateOptions): Promise<number
   const [repair] = await localDb
     .insert(localSchema.repairs)
     .values({
+      kind: 'repair',
       repairProviderId,
       repairDataSetId: null,
+      sourceProviderUrl: null,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
       targetDataSetId: null,

package/src/db/replicate-create.ts

@@ -69,8 +69,10 @@ export async function replicateCreate(options: ReplicateCreateOptions): Promise<
   const [repair] = await localDb
     .insert(localSchema.repairs)
     .values({
+      kind: 'replicate',
       repairProviderId: sourceProvider.providerId,
       repairDataSetId: sourceDataSet.dataSetId,
+      sourceProviderUrl: sourceProvider.serviceUrl,
       targetProviderId: targetProvider.providerId,
       targetProviderUrl: targetProvider.serviceUrl,
       targetDataSetId: null,

package/src/db/second-copy-create.ts

@@ -0,0 +1,147 @@
+import { readFile } from 'node:fs/promises'
+import { taskLog } from '@clack/prompts'
+import { getBlockNumber } from 'viem/actions'
+import { NoAlternateProviderError, RepairCreationError } from '../error.ts'
+import type { OperationInsert } from '../local-schema.ts'
+import type { Context } from '../types.ts'
+import { getRepairProvider } from './get-repair-provider.ts'
+
+const OPERATIONS_INSERT_BATCH_SIZE = 500
+
+export interface SecondCopyCreateOptions extends Context {
+  piecesFile: string
+  sourceProviderUrl: string
+  targetProviderId: bigint
+}
+
+type PiecesFile = {
+  aggregates: Array<{ piece_cid?: unknown }>
+}
+
+/**
+ * Normalize a provider service URL for storage and equality checks.
+ */
+export function normalizeProviderUrl(url: string): string {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    throw new RepairCreationError(`Invalid source provider URL: ${url}`)
+  }
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new RepairCreationError(`Source provider URL must be http(s): ${url}`)
+  }
+
+  // Drop trailing slash so endpoint comparisons are stable.
+  return parsed.toString().replace(/\/$/, '')
+}
+
+/**
+ * Parse a second-copy pieces file and return unique aggregate piece CIDs.
+ */
+export async function parsePieceCidsFromFile(piecesFile: string): Promise<string[]> {
+  let raw: unknown
+  try {
+    raw = JSON.parse(await readFile(piecesFile, 'utf8'))
+  } catch (error) {
+    throw new RepairCreationError(
+      error instanceof Error ? `Failed to read pieces file: ${error.message}` : 'Failed to read pieces file'
+    )
+  }
+
+  if (raw == null || typeof raw !== 'object' || !Array.isArray((raw as PiecesFile).aggregates)) {
+    throw new RepairCreationError('Pieces file must contain an "aggregates" array')
+  }
+
+  const cids = new Set<string>()
+  for (const aggregate of (raw as PiecesFile).aggregates) {
+    if (typeof aggregate?.piece_cid !== 'string' || aggregate.piece_cid.length === 0) {
+      throw new RepairCreationError('Each aggregate must include a non-empty "piece_cid" string')
+    }
+    cids.add(aggregate.piece_cid)
+  }
+
+  if (cids.size === 0) {
+    throw new RepairCreationError('Pieces file contains no piece CIDs')
+  }
+
+  return [...cids]
+}
+
+/**
+ * Prepare a second-copy job from a pieces file and an explicit source provider URL.
+ *
+ * @param options - The options for creating a second-copy job.
+ * @returns The ID of the created local job.
+ */
+export async function secondCopyCreate(options: SecondCopyCreateOptions): Promise<number> {
+  const { indexerDb, localDb, piecesFile, sourceProviderUrl, targetProviderId, client } = options
+  const localSchema = localDb._.fullSchema
+  const now = Date.now()
+  const blockNumber = await getBlockNumber(client)
+  const normalizedSourceUrl = normalizeProviderUrl(sourceProviderUrl)
+
+  const log = taskLog({
+    title: 'Creating second-copy',
+    limit: 10,
+    retainLog: true,
+  })
+
+  const targetProvider = await getRepairProvider({
+    indexerDb,
+    providerId: targetProviderId,
+  })
+
+  if (!targetProvider) {
+    throw new NoAlternateProviderError(targetProviderId)
+  }
+
+  const normalizedTargetUrl = normalizeProviderUrl(targetProvider.serviceUrl)
+  if (normalizedSourceUrl === normalizedTargetUrl) {
+    throw new RepairCreationError('Source provider URL must differ from the target provider URL')
+  }
+
+  const pieceCids = await parsePieceCidsFromFile(piecesFile)
+  log.message(`Loaded ${pieceCids.length} unique piece CIDs from ${piecesFile}`)
+
+  const [repair] = await localDb
+    .insert(localSchema.repairs)
+    .values({
+      kind: 'second_copy',
+      repairProviderId: 0n,
+      repairDataSetId: null,
+      sourceProviderUrl: normalizedSourceUrl,
+      targetProviderId: targetProvider.providerId,
+      targetProviderUrl: targetProvider.serviceUrl,
+      targetDataSetId: null,
+      blockNumber,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .returning({ id: localSchema.repairs.id })
+
+  if (!repair) throw new RepairCreationError('Failed to create second-copy row')
+
+  let totalOperations = 0
+  for (let offset = 0; offset < pieceCids.length; offset += OPERATIONS_INSERT_BATCH_SIZE) {
+    const batchCids = pieceCids.slice(offset, offset + OPERATIONS_INSERT_BATCH_SIZE)
+    const operations: OperationInsert[] = batchCids.map((cid) => ({
+      repairId: repair.id,
+      type: 'add_piece',
+      status: 'pending',
+      cid,
+      metadata: {},
+      alternateProvider: normalizedSourceUrl,
+      createdAt: now,
+      updatedAt: now,
+    }))
+
+    await localDb.insert(localSchema.operations).values(operations)
+    totalOperations += operations.length
+    log.message(`Inserted ${operations.length} operations (${totalOperations}/${pieceCids.length})`)
+  }
+
+  log.success(`Created second-copy ${repair.id} with ${totalOperations} operations`, { showLog: true })
+  return repair.id
+}