@filoz/repair-cli 0.1.1 → 0.2.0

package/src/pipeline/add-pieces.ts

@@ -0,0 +1,214 @@
+import { taskLog } from '@clack/prompts'
+import * as Piece from '@filoz/synapse-core/piece'
+import * as SP from '@filoz/synapse-core/sp'
+import { and, asc, eq, gt, inArray } from 'drizzle-orm'
+import PQueue from 'p-queue'
+import { dedupeCids } from '../db/dedupe-cids.ts'
+import { getTargetDataset } from '../db/get-target-dataset.ts'
+import { repairUpdate } from '../db/repair-update.ts'
+import { upsertOperations } from '../db/upsert-operations.ts'
+import type { OperationSelect, RepairSelect } from '../local-schema.ts'
+import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
+import { excludeOperationsByCid, hashLink, operationsToPullPieces } from '../utils.ts'
+
+export type RunPullPiecesPhaseOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  concurrency: number
+  batchSize: number
+  client: WalletClient
+}
+
+type CreateAddPiecesWorkerOptions = {
+  localDb: LocalDatabase
+  indexerDb: IndexerDatabase
+  repair: RepairSelect
+  client: WalletClient
+  state: {
+    totalBatches: number
+    totalOperations: number
+    completedOperations: number
+    failedOperations: number
+  }
+  log: ReturnType<typeof taskLog>
+}
+
+type AddPiecesJobOptions = {
+  operations: OperationSelect[]
+  batchNumber: number
+}
+
+/**
+ * Create a worker function that adds pieces to the target repair dataset.
+ */
+function createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log }: CreateAddPiecesWorkerOptions) {
+  return async (options: AddPiecesJobOptions) => {
+    let completedOps = 0
+    let failedOps = 0
+    let operations: OperationSelect[] = options.operations
+
+    const group = log.group(`Batch ${options.batchNumber}/${state.totalBatches}`)
+
+    try {
+      const dataset = await getTargetDataset({ localDb, repairId: repair.id, client })
+
+      // dedupe operations by CID on the target dataset for repairs jobs
+      if (repair.repairDataSetId == null) {
+        operations = await dedupeCids({ indexerDb, localDb, dataSetId: dataset.dataSetId, operations })
+      }
+      group.message(`Pulling ${operations.length} pieces...`)
+      // pull pieces
+      if (operations.length > 0) {
+        const pullResult = await SP.waitForPullPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: operationsToPullPieces(operations),
+          timeout: 1000 * 60 * 30,
+          onStatus: (status) => {
+            const completed = status.pieces.filter((piece) => piece.status === 'complete').length
+            const failed = status.pieces.filter((piece) => piece.status === 'failed').length
+            group.message(`Pull ${completed} completed, ${failed} failed`)
+          },
+        })
+
+        for (const { pieceCid, status } of pullResult.pieces) {
+          const cid = pieceCid.toString()
+          if (status !== 'complete') {
+            state.failedOperations++
+            failedOps++
+            const { operationToFailed, operationToCommit } = excludeOperationsByCid(operations, cid)
+            operations = operationToCommit
+            await upsertOperations({
+              localDb,
+              operations: operationToFailed.map((operation) => ({
+                ...operation,
+                status: 'failed',
+                error: `pull failed with status ${status}`,
+              })),
+            })
+          }
+        }
+      }
+
+      // add pieces
+      if (operations.length > 0) {
+        group.message(`Adding ${operations.length} pieces...`)
+        const addPiecesResult = await SP.addPieces(client, {
+          serviceURL: repair.targetProviderUrl,
+          dataSetId: dataset.dataSetId,
+          clientDataSetId: dataset.clientDataSetId,
+          pieces: operations.map((operation) => ({
+            pieceCid: Piece.from(operation.cid),
+            metadata: operation.metadata,
+          })),
+        })
+
+        group.message(`Waiting for add pieces ${hashLink(addPiecesResult.txHash, client.chain)}...`)
+        const addPiecesResult2 = await SP.waitForAddPieces(addPiecesResult)
+        state.completedOperations += operations.length
+        completedOps += operations.length
+        await upsertOperations({
+          localDb,
+          operations: operations.map((operation) => ({
+            ...operation,
+            status: 'completed',
+            error: null,
+            result: { dataSetId: addPiecesResult2.dataSetId, txHash: addPiecesResult2.txHash },
+          })),
+        })
+      }
+      group.message(`Done. ${completedOps} added, ${failedOps} failed`)
+    } catch (error) {
+      state.failedOperations += operations.length
+      const message = error instanceof Error ? error.message : 'Unknown error'
+      group.message(`${message.replace(/\n/g, ' ')}`)
+      await upsertOperations({
+        localDb,
+        operations: operations.map((operation) => ({
+          ...operation,
+          status: 'failed',
+          error: message,
+        })),
+      })
+    }
+  }
+}
+
+/**
+ * Add pieces to the target dataset.
+ */
+export async function runAddPieces({
+  localDb,
+  indexerDb,
+  repair,
+  concurrency,
+  batchSize,
+  client,
+}: RunPullPiecesPhaseOptions): Promise<void> {
+  const localSchema = localDb._.fullSchema
+  let cursor = 0
+
+  const totalOperations = await localDb.$count(
+    localSchema.operations,
+    and(
+      eq(localSchema.operations.repairId, repair.id),
+      eq(localSchema.operations.type, 'add_piece'),
+      inArray(localSchema.operations.status, ['pending', 'failed'])
+    )
+  )
+  let batchNumber = 0
+  const state = {
+    totalBatches: Math.ceil(totalOperations / batchSize),
+    totalOperations,
+    completedOperations: 0,
+    failedOperations: 0,
+  }
+
+  const log = taskLog({
+    title: 'Adding pieces',
+    limit: 1,
+  })
+
+  async function getNextBatch(): Promise<OperationSelect[] | null> {
+    const operations = await localDb.query.operations.findMany({
+      where: and(
+        eq(localSchema.operations.repairId, repair.id),
+        eq(localSchema.operations.type, 'add_piece'),
+        inArray(localSchema.operations.status, ['pending', 'failed']),
+        gt(localSchema.operations.id, cursor)
+      ),
+      orderBy: [asc(localSchema.operations.id)],
+      limit: batchSize,
+    })
+    if (operations.length === 0) {
+      return null
+    }
+
+    cursor = operations.at(-1)?.id ?? cursor
+    return operations
+  }
+
+  const addPiecesJob = createAddPiecesWorker({ localDb, indexerDb, repair, client, state, log })
+  const queue = new PQueue({ concurrency })
+
+  while (true) {
+    await queue.onSizeLessThan(concurrency)
+    const operations = await getNextBatch()
+    if (!operations) break
+    batchNumber++
+    const currentBatchNumber = batchNumber
+    queue.add(() => addPiecesJob({ operations, batchNumber: currentBatchNumber })).catch(console.error)
+  }
+
+  await queue.onIdle()
+
+  log.success(`Added ${state.completedOperations} pieces, ${state.failedOperations} failed`, { showLog: true })
+
+  await repairUpdate({
+    localDb,
+    repairId: repair.id,
+    status: state.failedOperations > 0 ? 'failed' : 'completed',
+  })
+}

package/src/pipeline/create-datasets.ts

@@ -1,13 +1,15 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPDPProvider } from '@filoz/synapse-core/sp-registry'
-import { getRepairDataset } from '../db/get-repair-dataset.ts'
+import { eq } from 'drizzle-orm'
+import { findRepairDataset } from '../db/find-repair-dataset.ts'
 import { repairUpdate } from '../db/repair-update.ts'
 import type { RepairSelect } from '../local-schema.ts'
 import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts'
-import { getRepairDatasetMetadata, hashLink } from '../utils.ts'
+import { hashLink } from '../utils.ts'
 
 export type EnsureRepairDatasetOptions = {
+  source: string
   localDb: LocalDatabase
   indexerDb: IndexerDatabase
   client: WalletClient
@@ -18,14 +20,8 @@ export type EnsureRepairDatasetOptions = {
  * Ensure the repair dataset exists by creating it if it doesn't.
  *
  * @param options - The options for ensuring the repair dataset.
- * @returns {Promise<bigint>} - The ID of the created dataset.
  */
-export async function ensureRepairDataset({
-  localDb,
-  indexerDb,
-  client,
-  repair,
-}: EnsureRepairDatasetOptions): Promise<bigint> {
+export async function ensureRepairDataset({ source, localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
   const log = p.taskLog({
     title: 'Ensuring repair dataset',
   })
@@ -37,10 +33,11 @@ export async function ensureRepairDataset({
 
   let datasetId: bigint | null = null
   // check if dataset already exists
-  const existingDatasetId = await getRepairDataset({
+  const existingDatasetId = await findRepairDataset({
     indexerDb,
     providerId: repair.targetProviderId,
     payer: client.account.address,
+    source,
   })
 
   if (existingDatasetId) {
@@ -52,7 +49,10 @@ export async function ensureRepairDataset({
       serviceURL: provider.pdp.serviceURL,
       payer: client.account.address,
       cdn: false,
-      metadata: getRepairDatasetMetadata(),
+      metadata: {
+        source,
+        withIPFSIndexing: '',
+      },
     })
     log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
     const waitForResult = await SP.waitForCreateDataSet({
@@ -68,3 +68,63 @@ export async function ensureRepairDataset({
   })
   return datasetId
 }
+
+/**
+ * Ensure a replication target dataset exists by creating a fresh dataset with source metadata.
+ *
+ * @param options - The options for ensuring the replication dataset.
+ */
+export async function ensureReplicateDataset({ localDb, indexerDb, client, repair }: EnsureRepairDatasetOptions) {
+  const log = p.taskLog({
+    title: 'Ensuring replication dataset',
+  })
+
+  if (repair.repairDataSetId == null) {
+    throw new Error('Missing source dataset ID')
+  }
+
+  const provider = await getPDPProvider(client, {
+    providerId: repair.targetProviderId,
+  })
+
+  if (!provider) throw new Error(`Target provider ${repair.targetProviderId} not found or inactive`)
+
+  if (repair.targetDataSetId != null) {
+    log.success(`Data set #${repair.targetDataSetId} already exists at ${provider.pdp.serviceURL}`)
+    return repair.targetDataSetId
+  }
+
+  const schema = indexerDb._.fullSchema
+  const sourceDataSet = await indexerDb.query.dataSets.findFirst({
+    where: eq(schema.dataSets.dataSetId, repair.repairDataSetId),
+    columns: {
+      metadata: true,
+      withCdn: true,
+    },
+  })
+
+  if (!sourceDataSet) {
+    throw new Error(`Source dataset ${repair.repairDataSetId} not found`)
+  }
+
+  const { txHash, statusUrl } = await SP.createDataSet(client, {
+    payee: provider.payee,
+    serviceURL: provider.pdp.serviceURL,
+    payer: client.account.address,
+    cdn: sourceDataSet.withCdn,
+    metadata: sourceDataSet.metadata ?? undefined,
+  })
+  log.message(`Waiting for data to be created at ${provider.pdp.serviceURL} ${hashLink(txHash, client.chain)}...`)
+  const waitForResult = await SP.waitForCreateDataSet({
+    statusUrl,
+  })
+  const datasetId = waitForResult.dataSetId
+  log.success(`Data set #${datasetId} created at ${provider.pdp.serviceURL}`)
+
+  await repairUpdate({
+    localDb,
+    repairId: repair.id,
+    targetDataSetId: datasetId,
+  })
+  return datasetId
+}

package/src/utils.ts

@@ -1,42 +1,36 @@
 import type { MetadataObject } from '@filoz/synapse-core'
 import { type Chain, getChain } from '@filoz/synapse-core/chains'
-import Conf from 'conf'
+import * as Piece from '@filoz/synapse-core/piece'
+import type * as SP from '@filoz/synapse-core/sp'
 import { getTableColumns, type SQL, sql } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
 import type { PgTable } from 'drizzle-orm/pg-core'
 import type { SQLiteTable } from 'drizzle-orm/sqlite-core'
 import { z } from 'incur'
+import { Conf } from 'iso-conf'
 import { request } from 'iso-web/http'
 import pLocate from 'p-locate'
 import terminalLink from 'terminal-link'
-import { createWalletClient, http } from 'viem'
+import { createWalletClient, type Hex, http } from 'viem'
 import { privateKeyToAccount } from 'viem/accounts'
 import packageJson from '../package.json' with { type: 'json' }
+import type { OperationSelect } from './local-schema.ts'
 import * as schema from './local-schema.ts'
-import type { Config, LocalDatabase } from './types.ts'
-
-export const EARLY_REPAIR_SOURCE = 'early-repair6'
+import type { LocalDatabase } from './types.ts'
+
+export const configSchema = z.object({
+  privateKey: z.string().optional(),
+  indexerMainnetUrl: z.url().optional(),
+  indexerCalibrationUrl: z.url().optional(),
+  chainId: z.number().optional(),
+  dbPath: z.string().optional(),
+  source: z.string().optional(),
+})
 
-export const config = new Conf<Config>({
+export const config = new Conf({
   projectName: packageJson.name,
   projectSuffix: '',
-  schema: {
-    privateKey: {
-      type: 'string',
-    },
-    dbPath: {
-      type: 'string',
-    },
-    indexerMainnetUrl: {
-      type: 'string',
-    },
-    indexerCalibrationUrl: {
-      type: 'string',
-    },
-    chainId: {
-      type: 'number',
-    },
-  },
+  schema: configSchema,
 })
 
 export const name = packageJson.name
@@ -47,7 +41,7 @@ function privateKeyFromConfig() {
   if (!privateKey) {
     throw new Error('Private key not found. Please run `repair-cli setup` first.')
   }
-  return privateKey
+  return privateKey as Hex
 }
 
 /**
@@ -102,11 +96,19 @@ export async function migrateLocalDatabase(db: LocalDatabase) {
       target_provider_id text NOT NULL,
       target_provider_url text NOT NULL,
       target_data_set_id text,
+      repair_data_set_id text,
       block_number text NOT NULL,
       created_at integer NOT NULL,
       updated_at integer NOT NULL
     )
   `)
+  try {
+    await db.$client.execute(`
+      ALTER TABLE repairs ADD COLUMN repair_data_set_id text
+    `)
+  } catch {
+    // Column already exists on databases created with the updated schema.
+  }
   await db.$client.execute(`
     CREATE TABLE IF NOT EXISTS operations (
       id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
@@ -137,14 +139,6 @@ export function hashLink(hash: string, chain: Chain) {
   return link
 }
 
-/** Get metadata for the single IPFS-enabled repair dataset. */
-export function getRepairDatasetMetadata(): MetadataObject {
-  return {
-    source: EARLY_REPAIR_SOURCE,
-    withIPFSIndexing: '',
-  }
-}
-
 /**
  * Get a piece from a service URL
  */
@@ -207,3 +201,41 @@ export const buildConflictUpdateColumns = <T extends PgTable | SQLiteTable, Q ex
 
   return r
 }
+
+/**
+ * Filter out operations by CID
+ */
+export function excludeOperationsByCid(operations: OperationSelect[], cid: string) {
+  const operationToCommit: OperationSelect[] = []
+  const operationToFailed: OperationSelect[] = []
+  for (const operation of operations) {
+    if (operation.cid === cid) {
+      operationToFailed.push(operation)
+    } else {
+      operationToCommit.push(operation)
+    }
+  }
+  return { operationToCommit, operationToFailed }
+}
+
+/**
+ * Convert operations to pull pieces deduped by CID
+ */
+export function operationsToPullPieces(operations: OperationSelect[]) {
+  const cids = new Set<string>()
+  const pieces: SP.PullPieceInput[] = []
+  for (const operation of operations) {
+    if (cids.has(operation.cid)) {
+      continue
+    }
+    pieces.push({
+      sourceUrl: Piece.createPieceUrlPDP({
+        cid: operation.cid,
+        serviceURL: operation.alternateProvider,
+      }),
+      pieceCid: Piece.from(operation.cid),
+    })
+    cids.add(operation.cid)
+  }
+  return pieces
+}

package/dist/src/db/get-providers-by-cid.d.ts

@@ -1,10 +0,0 @@
-import type { IndexerDatabase, RepairProvider } from '../types.ts';
-export type GetProvidersByCidOptions = {
-    indexerDb: IndexerDatabase;
-    cids: readonly string[];
-    excludedProviderIds: readonly bigint[];
-    blockNumber: bigint;
-};
-export type ProvidersByCid = Record<string, RepairProvider[]>;
-export declare function getProvidersByCid({ indexerDb, cids, excludedProviderIds, blockNumber, }: GetProvidersByCidOptions): Promise<ProvidersByCid>;
-//# sourceMappingURL=get-providers-by-cid.d.ts.map

package/dist/src/db/get-providers-by-cid.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"get-providers-by-cid.d.ts","sourceRoot":"","sources":["../../../src/db/get-providers-by-cid.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,MAAM,wBAAwB,GAAG;IACrC,SAAS,EAAE,eAAe,CAAA;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,mBAAmB,EAAE,SAAS,MAAM,EAAE,CAAA;IACtC,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAKD,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAA;AAS7D,wBAAsB,iBAAiB,CAAC,EACtC,SAAS,EACT,IAAI,EACJ,mBAAmB,EACnB,WAAW,GACZ,EAAE,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC,CA+CpD"}

package/dist/src/db/get-providers-by-cid.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"get-providers-by-cid.js","sourceRoot":"","sources":["../../../src/db/get-providers-by-cid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,aAAa,CAAA;AAsBhF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,SAAS,EACT,IAAI,EACJ,mBAAmB,EACnB,WAAW,GACc;IACzB,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAmB,CAAA;IACzF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE5C,MAAM,OAAO,GAAG;QACd,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACrC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACtF,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC;KAEjC,CAAA;IACD,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAA;IAChF,CAAC;IAGD,MAAM,IAAI,GAAG,MAAM,SAAS;SACzB,cAAc,CAAC;QACd,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,eAAe;QACjD,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;QACvC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;QACnC,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ;KACpC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SACnB,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SAClF,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;SACxF,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;SACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;IAEpE,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrE,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,eAAe,EAAE,QAAQ,CAAC,eAAe;gBACzC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAA;AACvB,CAAC"}

package/dist/src/db/get-repair-dataset.d.ts

@@ -1,9 +0,0 @@
-import type { Address } from 'viem';
-import type { IndexerDatabase } from '../types.ts';
-export type GetRepairDatasetOptions = {
-    indexerDb: IndexerDatabase;
-    providerId: bigint;
-    payer: Address;
-};
-export declare function getRepairDataset({ indexerDb, providerId, payer, }: GetRepairDatasetOptions): Promise<bigint | null>;
-//# sourceMappingURL=get-repair-dataset.d.ts.map

package/dist/src/db/get-repair-dataset.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"get-repair-dataset.d.ts","sourceRoot":"","sources":["../../../src/db/get-repair-dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AACnC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAGlD,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,eAAe,CAAA;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAOD,wBAAsB,gBAAgB,CAAC,EACrC,SAAS,EACT,UAAU,EACV,KAAK,GACN,EAAE,uBAAuB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAuBlD"}

package/dist/src/db/get-repair-dataset.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"get-repair-dataset.js","sourceRoot":"","sources":["../../../src/db/get-repair-dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAGlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAajD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EACrC,SAAS,EACT,UAAU,EACV,KAAK,GACmB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IAErC,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS;SAC1B,MAAM,CAAC;QACN,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;KACrC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;SACrB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,EAC1C,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,EAClC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EACnC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,EAC9C,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAC,EAC/C,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,CAAC,EAClC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAC3C,CACF;SACA,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;SACvC,KAAK,CAAC,CAAC,CAAC,CAAA;IAEX,OAAO,GAAG,CAAC,SAAS,IAAI,IAAI,CAAA;AAC9B,CAAC"}

package/dist/src/db/sync-pieces-onchain.d.ts

@@ -1,10 +0,0 @@
-import type { OperationSelect } from '../local-schema.ts';
-import type { IndexerDatabase, LocalDatabase } from '../types.ts';
-export type SyncPiecesOnchainOptions = {
-    indexerDb: IndexerDatabase;
-    localDb: LocalDatabase;
-    dataSetId: bigint;
-    cidToOperation: Map<string, OperationSelect>;
-};
-export declare function syncPiecesOnchain({ indexerDb, localDb, dataSetId, cidToOperation }: SyncPiecesOnchainOptions): Promise<number>;
-//# sourceMappingURL=sync-pieces-onchain.d.ts.map

package/dist/src/db/sync-pieces-onchain.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sync-pieces-onchain.d.ts","sourceRoot":"","sources":["../../../src/db/sync-pieces-onchain.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAmB,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAGjE,MAAM,MAAM,wBAAwB,GAAG;IACrC,SAAS,EAAE,eAAe,CAAA;IAC1B,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;CAC7C,CAAA;AAKD,wBAAsB,iBAAiB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,wBAAwB,mBAqClH"}

package/dist/src/db/sync-pieces-onchain.js

@@ -1,35 +0,0 @@
-import { and, eq, inArray } from 'drizzle-orm';
-import { upsertOperations } from "./upsert-operations.js";
-export async function syncPiecesOnchain({ indexerDb, localDb, dataSetId, cidToOperation }) {
-    const cids = Array.from(cidToOperation.keys());
-    const schema = indexerDb._.fullSchema;
-    let completedOperations = 0;
-    const rows = await indexerDb
-        .select({ cid: schema.pieces.cid })
-        .from(schema.pieces)
-        .where(and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false), inArray(schema.pieces.cid, cids)));
-    const existingCids = new Set();
-    const completedOperation = [];
-    for (const row of rows) {
-        const operation = cidToOperation.get(row.cid);
-        if (!operation) {
-            continue;
-        }
-        completedOperation.push({
-            ...operation,
-            status: 'completed',
-            error: null,
-        });
-        existingCids.add(row.cid);
-        cidToOperation.delete(row.cid);
-        completedOperations++;
-    }
-    if (completedOperation.length > 0) {
-        await upsertOperations({
-            localDb,
-            operations: completedOperation,
-        });
-    }
-    return completedOperations;
-}
-//# sourceMappingURL=sync-pieces-onchain.js.map

package/dist/src/db/sync-pieces-onchain.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"sync-pieces-onchain.js","sourceRoot":"","sources":["../../../src/db/sync-pieces-onchain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAG9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AAYzD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAA4B;IACjH,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;IACrC,IAAI,mBAAmB,GAAG,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,MAAM,SAAS;SACzB,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;SAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;SACnB,KAAK,CACJ,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAChH,CAAA;IAEH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;IACtC,MAAM,kBAAkB,GAAsB,EAAE,CAAA;IAEhD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAQ;QACV,CAAC;QACD,kBAAkB,CAAC,IAAI,CAAC;YACtB,GAAG,SAAS;YACZ,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAA;QACF,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACzB,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC9B,mBAAmB,EAAE,CAAA;IACvB,CAAC;IAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,gBAAgB,CAAC;YACrB,OAAO;YACP,UAAU,EAAE,kBAAkB;SAC/B,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,mBAAmB,CAAA;AAC5B,CAAC"}

package/dist/src/pipeline/pull.d.ts

@@ -1,30 +0,0 @@
-import { taskLog } from '@clack/prompts';
-import type { OperationSelect, RepairSelect } from '../local-schema.ts';
-import type { IndexerDatabase, LocalDatabase, WalletClient } from '../types.ts';
-export type PullPiecesBatch = {
-    operations: OperationSelect[];
-};
-export type RunPullPiecesPhaseOptions = {
-    localDb: LocalDatabase;
-    indexerDb: IndexerDatabase;
-    repair: RepairSelect;
-    concurrency: number;
-    batchSize: number;
-    client: WalletClient;
-    reset: boolean;
-};
-export declare function createPullPiecesWorker({ localDb, indexerDb, repair, client, state, log, }: {
-    localDb: LocalDatabase;
-    indexerDb: IndexerDatabase;
-    repair: RepairSelect;
-    client: WalletClient;
-    state: {
-        totalBatches: number;
-        totalOperations: number;
-        completedOperations: number;
-        failedOperations: number;
-    };
-    log: ReturnType<typeof taskLog>;
-}): (batch: PullPiecesBatch, batchNumber: number) => Promise<void>;
-export declare function runPullPiecesPhase({ localDb, indexerDb, repair, concurrency, batchSize, client, reset, }: RunPullPiecesPhaseOptions): Promise<void>;
-//# sourceMappingURL=pull.d.ts.map

package/dist/src/pipeline/pull.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/pipeline/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AAUxC,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAI/E,MAAM,MAAM,eAAe,GAAG;IAC5B,UAAU,EAAE,eAAe,EAAE,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE,OAAO,CAAA;CACf,CAAA;AAGD,wBAAgB,sBAAsB,CAAC,EACrC,OAAO,EACP,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,EACL,GAAG,GACJ,EAAE;IACD,OAAO,EAAE,aAAa,CAAA;IACtB,SAAS,EAAE,eAAe,CAAA;IAC1B,MAAM,EAAE,YAAY,CAAA;IACpB,MAAM,EAAE,YAAY,CAAA;IACpB,KAAK,EAAE;QACL,YAAY,EAAE,MAAM,CAAA;QACpB,eAAe,EAAE,MAAM,CAAA;QACvB,mBAAmB,EAAE,MAAM,CAAA;QAC3B,gBAAgB,EAAE,MAAM,CAAA;KACzB,CAAA;IACD,GAAG,EAAE,UAAU,CAAC,OAAO,OAAO,CAAC,CAAA;CAChC,IACe,OAAO,eAAe,EAAE,aAAa,MAAM,mBA+H1D;AAQD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,EACX,SAAS,EACT,MAAM,EACN,KAAK,GACN,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6D3C"}