@filoz/repair-cli 0.4.0 → 1.0.0

package/src/pipeline/add-pieces.ts

@@ -3,17 +3,18 @@ import * as Piece from '@filoz/synapse-core/piece'
 import * as SP from '@filoz/synapse-core/sp'
 import { and, asc, eq, gt, inArray } from 'drizzle-orm'
 import PQueue from 'p-queue'
+import { listDataSetPieces } from '../api.ts'
 import { dedupeCids } from '../db/dedupe-cids.ts'
 import { getTargetDataset } from '../db/get-target-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import { upsertOperations } from '../db/upsert-operations.ts'
 import type { OperationSelect, RepairSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
 import { completeConfirmedOperations, excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
 
 export type RunPullPiecesPhaseOptions = {
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   repair: RepairSelect
   concurrency: number
   batchSize: number
@@ -22,7 +23,8 @@ export type RunPullPiecesPhaseOptions = {
 
 type CreateAddPiecesWorkerOptions = {
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  /** CIDs already in the target dataset; repairs only. Updated as batches are added. */
+  targetCids?: Set<string>
   repair: RepairSelect
   client: WalletClient
   state: {
@@ -42,12 +44,11 @@ type AddPiecesJobOptions = {
 /**
  * Create a worker function that adds pieces to the target repair dataset.
  */
-function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
+function createAddPiecesWorker({ localDb, targetCids, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
   return async (options: AddPiecesJobOptions) => {
     let completedOps = 0
     let failedOps = 0
     let operations: OperationSelect[] = options.operations
-    const isRepair = repair.repairDataSetId == null
 
     const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
 
@@ -55,8 +56,8 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
       const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
 
       // dedupe operations by CID on the target dataset for repairs jobs
-      if (isRepair) {
-        operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
+      if (targetCids) {
+        operations = await dedupeCids({ localDb, existingCids: targetCids, operations })
       }
       // A previous run may have submitted a transaction and crashed before marking rows completed.
       const operationsBeforeConfirmationCheck = operations.length
@@ -111,7 +112,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
           clientDataSetId: dataset.clientDataSetId,
           pieces: operations.map((operation) => ({
             pieceCid: Piece.from(operation.cid),
-            metadata: isRepair ? undefined : operation.metadata,
+            metadata: targetCids ? undefined : operation.metadata,
           })),
         })
         await upsertOperations({
@@ -126,6 +127,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
         const waitForAddPieces = await SP.waitForAddPieces(addPiecesResult)
         state.completedOperations += operations.length
         completedOps += operations.length
+        for (const operation of operations) targetCids?.add(operation.cid)
         await upsertOperations({
           localDb,
           operations: operations.map((operation) => ({
@@ -158,7 +160,7 @@ function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log
  */
 export async function runAddPieces({
   localDb,
-  indexerDb,
+  api,
   repair,
   concurrency,
   batchSize,
@@ -188,6 +190,14 @@ export async function runAddPieces({
     limit: 1,
   })
 
+  // Repairs and second copies dedupe into one shared dataset; read its CIDs once instead of per batch.
+  let targetCids: Set<string> | undefined
+  if (repair.repairDataSetId == null && totalOperations > 0) {
+    const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
+    const pieces = await listDataSetPieces(api, dataset.dataSetId, { removed: false })
+    targetCids = new Set(pieces.flatMap((piece) => (piece.cid == null ? [] : [piece.cid])))
+  }
+
   async function getNextBatch(): Promise<OperationSelect[] | null> {
     const operations = await localDb.query.operations.findMany({
       where: and(
@@ -207,7 +217,7 @@ export async function runAddPieces({
     return operations
   }
 
-  const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log })
+  const addPiecesJob = createAddPiecesWorker({ localDb, targetCids, repair, client, state, log })
   const queue = new PQueue({ concurrency })
 
   while (true) {

package/src/pipeline/create-datasets.ts

@@ -1,18 +1,18 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
-import { eq } from 'drizzle-orm'
 import type { Address } from 'viem'
+import { getDataSet, toStringMetadata } from '../api.ts'
 import { findRepairDataset } from '../db/find-repair-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import type { RepairSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import type { IndexerApi, LocalDatabase, WalletClient } from '../types.ts'
 import { hashLink } from '../utils.ts'
 
 export type EnsureRepairDatasetOptions = {
   source: string
   localDb: LocalDatabase
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   client: WalletClient
   repair: RepairSelect
   payer: Address
@@ -23,14 +23,7 @@ export type EnsureRepairDatasetOptions = {
  *
  * @param options - The options for ensuring the repair dataset.
  */
-export async function ensureRepairDataset({
-  source,
-  localDb,
-  indexerDb,
-  client,
-  repair,
-  payer,
-}: EnsureRepairDatasetOptions) {
+export async function ensureRepairDataset({ source, localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring repair dataset',
   })
@@ -43,7 +36,7 @@ export async function ensureRepairDataset({
   let datasetId: bigint | null = null
   // check if dataset already exists
   const existingDatasetId = await findRepairDataset({
-    indexerDb,
+    api,
     providerId: repair.targetProviderId,
     payer,
     source,
@@ -83,13 +76,7 @@ export async function ensureRepairDataset({
  *
  * @param options - The options for ensuring the replication dataset.
  */
-export async function ensureReplicateDataset({
-  localDb,
-  indexerDb,
-  client,
-  repair,
-  payer,
-}: EnsureRepairDatasetOptions) {
+export async function ensureReplicateDataset({ localDb, api, client, repair, payer }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring replication dataset',
   })
@@ -109,14 +96,7 @@ export async function ensureReplicateDataset({
     return repair.targetDataSetId
   }
 
-  const schema = indexerDb._.fullSchema
-  const sourceDataSet = await indexerDb.query.dataSets.findFirst({
-    where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
-    columns: {
-      metadata: true,
-      withCdn: true,
-    },
-  })
+  const sourceDataSet = await getDataSet(api, repair.repairDataSetId)
 
   if (!sourceDataSet) {
     throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
@@ -126,8 +106,8 @@ export async function ensureReplicateDataset({
     payee: provider.payee,
     serviceURL: provider.pdp.serviceURL,
     payer,
-    cdn: sourceDataSet.withCdn,
-    metadata: sourceDataSet.metadata ?? undefined,
+    cdn: sourceDataSet.withCdn ?? false,
+    metadata: sourceDataSet.metadata ? toStringMetadata(sourceDataSet.metadata) : undefined,
   })
   log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
   const waitForResult = await SP.waitForCreateDataSet({

package/src/types.ts

@@ -1,25 +1,27 @@
+import type { ApiClient } from '@filecoin-project/api-client'
 import type { Chain } from '@filoz/synapse-core/chains'
 import type { Client } from '@libsql/client'
 import type { LibSQLDatabase } from 'drizzle-orm/libsql'
-import type { NodePgDatabase } from 'drizzle-orm/node-postgres'
 import type { z } from 'incur'
-import type { Pool } from 'pg'
 import type { Account, Address, Hex, Transport, Client as ViemClient } from 'viem'
-import type * as indexerSchema from './indexer-schema.ts'
+import type { Network } from './api.ts'
 import type * as localSchema from './local-schema.ts'
 import type { contextSchema } from './middleware.ts'
 export type LocalDatabase = LibSQLDatabase<typeof localSchema> & {
   $client: Client
 }
 
-export type IndexerDatabase = NodePgDatabase<typeof indexerSchema> & {
-  $client: Pool
+/**
+ * filecoin-api client bound to the configured network.
+ */
+export type IndexerApi = {
+  client: ApiClient
+  network: Network
 }
 
 export interface Config {
   privateKey: Hex
-  indexerMainnetUrl: string
-  indexerCalibrationUrl: string
+  apiUrl: string
   chainId: number
   dbPath: string
 }

package/src/utils.ts

@@ -3,7 +3,6 @@ import * as Piece from '@filoz/synapse-core/piece'
 import type * as SP from '@filoz/synapse-core/sp'
 import { getTableColumns, inArray, type SQL, sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
-import type { PgTable } from 'drizzle-orm/pg-core'
 import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
 import { z } from 'incur'
 import { Conf } from 'iso-conf'
@@ -20,8 +19,7 @@ import type { LocalDatabase, WalletClient } from './types.ts'
 
 export const configSchema = z.object({
   privateKey: z.string().optional(),
-  indexerMainnetUrl: z.url().optional(),
-  indexerCalibrationUrl: z.url().optional(),
+  apiUrl: z.url().optional(),
   chainId: z.number().optional(),
   dbPath: z.string().optional(),
   source: z.string().optional(),
@@ -273,7 +271,7 @@ export async function findPieceOnProviders(providers: string[], pieceCid: string
   return result
 }
 
-export const buildConflictUpdateColumns = <T extends PgTable | SQLiteTable, Q extends keyof T['_']['columns']>(
+export const buildConflictUpdateColumns = <T extends SQLiteTable, Q extends keyof T['_']['columns']>(
   table: T,
   columns?: Q[]
 ) => {

package/dist/src/db/find-providers-by-cid.d.ts

@@ -1,9 +0,0 @@
-import type { IndexerDatabase, RepairProvider } from '../types.ts';
-export type FindProvidersByCidOptions = {
-    indexerDb: IndexerDatabase;
-    cids: readonly string[];
-    blockNumber: bigint;
-};
-export type ProvidersByCid = Record<string, RepairProvider[]>;
-export declare function findProvidersByCid({ indexerDb, cids, blockNumber, }: FindProvidersByCidOptions): Promise<ProvidersByCid>;
-//# sourceMappingURL=find-providers-by-cid.d.ts.map

package/dist/src/db/find-providers-by-cid.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"find-providers-by-cid.d.ts","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,EAAE,eAAe,CAAA;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAKD,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAA;AAS7D,wBAAsB,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACZ,EAAE,yBAAyB,GAAG,OAAO,CAAC,cAAc,CAAC,CA4CrD"}

package/dist/src/db/find-providers-by-cid.js

@@ -1,43 +0,0 @@
-import { and, asc, eq, inArray, isNull, lte, or } from 'drizzle-orm';
-export async function findProvidersByCid({ indexerDb, cids, blockNumber, }) {
-    const schema = indexerDb._.fullSchema;
-    const providersByCid = Object.fromEntries(cids.map((cid) => [cid, []]));
-    if (cids.length === 0)
-        return providersByCid;
-    const filters = [
-        inArray(schema.pieces.cid, [...cids]),
-        eq(schema.dataSets.deleted, false),
-        or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
-        eq(schema.pieces.removed, false),
-        or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)),
-    ];
-    const rows = await indexerDb
-        .selectDistinct({
-        cid: schema.pieces.cid,
-        providerId: schema.providers.providerId,
-        providerAddress: schema.providers.providerAddress,
-        name: schema.providers.name,
-        serviceUrl: schema.providers.serviceUrl,
-        approved: schema.providers.approved,
-        endorsed: schema.providers.endorsed,
-    })
-        .from(schema.pieces)
-        .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
-        .innerJoin(schema.providers, eq(schema.dataSets.providerId, schema.providers.providerId))
-        .where(and(...filters))
-        .orderBy(asc(schema.pieces.cid), asc(schema.providers.providerId));
-    for (const { cid, ...provider } of rows) {
-        if (provider.providerAddress && provider.serviceUrl && provider.name) {
-            providersByCid[cid]?.push({
-                providerId: provider.providerId,
-                providerAddress: provider.providerAddress,
-                name: provider.name,
-                serviceUrl: provider.serviceUrl,
-                approved: provider.approved,
-                endorsed: provider.endorsed,
-            });
-        }
-    }
-    return providersByCid;
-}
-//# sourceMappingURL=find-providers-by-cid.js.map

package/dist/src/db/find-providers-by-cid.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"find-providers-by-cid.js","sourceRoot":"","sources":["../../../src/db/find-providers-by-cid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAqBpE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,SAAS,EACT,IAAI,EACJ,WAAW,GACe;IAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAmB,CAAA;IACzF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE5C,MAAM,OAAO,GAAG;QACd,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtF,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;KAC7E,CAAA;IAGD,MAAM,IAAI,GAAG,MAAM,SAAS;SACzB,cAAc,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,eAAe;QACjD,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;QACnC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;KACpC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SACnB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SAClF,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACxF,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;SACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrE,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAA;AACvB,CAAC"}