@filoz/repair-cli 0.3.0 → 1.0.0

package/src/cli.ts

@@ -4,6 +4,7 @@ import { datasets } from './commands/datasets.ts'
 import { providers } from './commands/providers.ts'
 import { repair } from './commands/repair.ts'
 import { replicate } from './commands/replicate.ts'
+import { secondCopy } from './commands/second-copy.ts'
 import { sessionKey } from './commands/session-key.ts'
 import { setup } from './commands/setup.ts'
 import { wallet } from './commands/wallet.ts'
@@ -18,6 +19,7 @@ cli.command(setup)
 cli.command(wallet)
 cli.command(repair)
 cli.command(replicate)
+cli.command(secondCopy)
 cli.command(datasets)
 cli.command(providers)
 cli.command(sessionKey)

package/src/commands/datasets.ts

@@ -1,9 +1,9 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPdpDataSet } from '@filoz/synapse-core/warm-storage'
-import { and, asc, eq } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { isAddress } from 'viem'
+import { getDataSet, getProvider, listDataSetPieces, listDataSets, listProviders } from '../api.ts'
 import { contextMiddleware, contextSchema } from '../middleware.ts'
 import { globalOptions, hashLink } from '../utils.ts'
 
@@ -18,42 +18,33 @@ datasets.command('list', {
   options: globalOptions.extend({
     providerId: z.coerce.bigint().optional().describe('Filter datasets by provider ID'),
     payer: z.string().refine(isAddress, 'Invalid address').optional().describe('Filter datasets by payer address'),
+    stats: z.boolean().optional().default(false).describe('Include piece counts (slow: walks every piece)'),
   }),
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const schema = c.var.indexerDb._.fullSchema
+      const { api } = c.var
       const payer = c.options.payer?.toLowerCase() ?? c.var.client.account.address.toLowerCase()
-      const conditions = [eq(schema.dataSets.deleted, false), eq(schema.dataSets.payer, payer)]
+      const [rows, providers] = await Promise.all([
+        listDataSets(api, { owner: payer, providerId: c.options.providerId, deleted: false }),
+        listProviders(api),
+      ])
+      const serviceUrls = new Map(providers.map((provider) => [provider.providerId, provider.serviceUrl]))
 
-      if (c.options.providerId != null) {
-        conditions.push(eq(schema.dataSets.providerId, c.options.providerId))
-      }
-
-      const datasets = await c.var.indexerDb.query.dataSets.findMany({
-        where: and(...conditions),
-        with: {
-          provider: true,
-          pieces: {
-            where: eq(c.var.indexerDb._.fullSchema.pieces.removed, false),
-          },
-        },
-        orderBy: [asc(schema.dataSets.dataSetId)],
-      })
-
-      const datasetsFlattened = datasets.map((dataset) => {
-        const { provider, pieces } = dataset
-        return {
+      const datasetsFlattened = []
+      for (const dataset of rows.reverse()) {
+        datasetsFlattened.push({
           id: dataset.dataSetId,
           withCdn: dataset.withCdn,
           withIpfsIndexing: dataset.withIpfsIndexing,
           source: dataset.source,
-          provider: provider.serviceUrl,
+          provider: dataset.providerId == null ? null : serviceUrls.get(dataset.providerId),
           pdpEndEpoch: dataset.pdpEndEpoch,
-          pieces: pieces.length,
-          //   metadata: JSON.stringify(dataset.metadata),
-        }
-      })
+          ...(c.options.stats && {
+            pieces: (await listDataSetPieces(api, BigInt(dataset.dataSetId), { removed: false })).length,
+          }),
+        })
+      }
 
       return c.ok({
         datasets: datasetsFlattened,
@@ -138,15 +129,8 @@ datasets.command('show', {
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const dataset = await c.var.indexerDb.query.dataSets.findFirst({
-        where: eq(c.var.indexerDb._.fullSchema.dataSets.dataSetId, c.args.id),
-        with: {
-          provider: true,
-          pieces: {
-            where: eq(c.var.indexerDb._.fullSchema.pieces.removed, false),
-          },
-        },
-      })
+      const { api } = c.var
+      const dataset = await getDataSet(api, c.args.id)
 
       if (!dataset) {
         return c.error({
@@ -155,7 +139,11 @@ datasets.command('show', {
           retryable: false,
         })
       }
-      return c.ok(dataset)
+      const [provider, pieces] = await Promise.all([
+        dataset.providerId == null ? null : getProvider(api, BigInt(dataset.providerId)),
+        listDataSetPieces(api, c.args.id, { removed: false }),
+      ])
+      return c.ok({ ...dataset, provider, pieces: pieces.reverse() })
     } catch (error) {
       if (c.options.debug) {
         console.error(error)

package/src/commands/providers.ts

@@ -1,6 +1,7 @@
-import { and, asc, count, eq, inArray, isNull, lte, or, type SQLWrapper, sum } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { getBlockNumber } from 'viem/actions'
+import { listProviders, providerPiecePages } from '../api.ts'
+import { getActiveDataSetIds } from '../db/get-active-data-set-ids.ts'
 import { contextMiddleware, contextSchema } from '../middleware.ts'
 import { globalOptions } from '../utils.ts'
 
@@ -12,6 +13,13 @@ function formatBytesAsGb(bytes: bigint): string {
   return `${whole}.${fraction.toString().padStart(2, '0')} GB`
 }
 
+/** Compare decimal ID strings numerically. */
+function compareIds(a: string, b: string): number {
+  const x = BigInt(a)
+  const y = BigInt(b)
+  return x < y ? -1 : x > y ? 1 : 0
+}
+
 export const providers = Cli.create('providers', {
   description: 'Provider commands',
   options: globalOptions,
@@ -22,51 +30,36 @@ providers.command('list', {
   description: 'List all providers from the indexer',
   options: globalOptions.extend({
     all: z.boolean().optional().default(false).describe('Include all providers'),
+    stats: z
+      .boolean()
+      .optional()
+      .default(false)
+      .describe('Include piece count and total size (slow: walks every piece of each provider)'),
   }),
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const schema = c.var.indexerDb._.fullSchema
-      const filters: (SQLWrapper | undefined)[] = [
-        eq(schema.providers.providerActive, true),
-        eq(schema.providers.pdpProductActive, true),
-      ]
-      if (!c.options.all) {
-        filters.push(or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)))
-      }
-      const blockNumber = await getBlockNumber(c.var.client)
-      const rows = await c.var.indexerDb.query.providers.findMany({
-        orderBy: [asc(schema.providers.providerId)],
-        where: and(...filters),
-      })
-
-      const providerIds = rows.map((provider) => provider.providerId)
-      const statsByProviderId = new Map<bigint, { pieceCount: number; totalSize: bigint }>()
+      const { api } = c.var
+      const rows = (await listProviders(api, { active: true }))
+        .filter((provider) => provider.pdpProductActive && (c.options.all || provider.approved || provider.endorsed))
+        .sort((a, b) => compareIds(a.providerId, b.providerId))
 
-      if (providerIds.length > 0) {
-        const stats = await c.var.indexerDb
-          .select({
-            providerId: schema.dataSets.providerId,
-            pieceCount: count(schema.pieces.pieceId),
-            totalSize: sum(schema.pieces.rawSize),
-          })
-          .from(schema.pieces)
-          .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
-          .where(
-            and(
-              inArray(schema.dataSets.providerId, providerIds),
-              eq(schema.dataSets.deleted, false),
-              or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
-              eq(schema.pieces.removed, false)
-            )
-          )
-          .groupBy(schema.dataSets.providerId)
+      const statsByProviderId = new Map<string, { pieceCount: number; totalSize: bigint }>()
 
-        for (const stat of stats) {
-          statsByProviderId.set(stat.providerId, {
-            pieceCount: stat.pieceCount,
-            totalSize: stat.totalSize == null ? 0n : BigInt(stat.totalSize),
-          })
+      if (c.options.stats && rows.length > 0) {
+        const blockNumber = await getBlockNumber(c.var.client)
+        for (const provider of rows) {
+          const providerId = BigInt(provider.providerId)
+          const activeDataSetIds = await getActiveDataSetIds({ api, providerId, blockNumber })
+          const stats = { pieceCount: 0, totalSize: 0n }
+          for await (const pieces of providerPiecePages(api, providerId, { removed: false })) {
+            for (const piece of pieces) {
+              if (!activeDataSetIds.has(piece.dataSetId)) continue
+              stats.pieceCount++
+              stats.totalSize += BigInt(piece.rawSize ?? 0)
+            }
+          }
+          statsByProviderId.set(provider.providerId, stats)
         }
       }
 
@@ -78,8 +71,7 @@ providers.command('list', {
           serviceUrl: provider.serviceUrl,
           approved: provider.approved,
           endorsed: provider.endorsed,
-          pieceCount: stats?.pieceCount ?? 0,
-          totalSize: formatBytesAsGb(stats?.totalSize ?? 0n),
+          ...(stats && { pieceCount: stats.pieceCount, totalSize: formatBytesAsGb(stats.totalSize) }),
         }
       })
 

package/src/commands/repair.ts

@@ -1,4 +1,4 @@
-import { and, desc, eq, inArray, isNull } from 'drizzle-orm'
+import { and, desc, eq, inArray } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { isAddress } from 'viem'
 import { repairCreate } from '../db/repair-create.ts'
@@ -52,7 +52,7 @@ repair.command('list', {
     try {
       const localSchema = c.var.localDb._.fullSchema
       const repairs = await c.var.localDb.query.repairs.findMany({
-        where: isNull(localSchema.repairs.repairDataSetId),
+        where: eq(localSchema.repairs.kind, 'repair'),
         orderBy: [desc(localSchema.repairs.createdAt)],
         with: {
           operations: true,
@@ -147,7 +147,7 @@ repair.command('run', {
         where: and(
           eq(schema.repairs.id, c.args.repairId),
           inArray(schema.repairs.status, ['pending', 'failed']),
-          isNull(schema.repairs.repairDataSetId)
+          eq(schema.repairs.kind, 'repair')
         ),
       })
       if (!repair) {

package/src/commands/replicate.ts

@@ -1,4 +1,4 @@
-import { and, desc, eq, inArray, isNotNull } from 'drizzle-orm'
+import { and, desc, eq, inArray } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { isAddress } from 'viem'
 import { repairDelete } from '../db/repair-delete.ts'
@@ -52,7 +52,7 @@ replicate.command('list', {
     try {
       const localSchema = c.var.localDb._.fullSchema
       const replications = await c.var.localDb.query.repairs.findMany({
-        where: isNotNull(localSchema.repairs.repairDataSetId),
+        where: eq(localSchema.repairs.kind, 'replicate'),
         orderBy: [desc(localSchema.repairs.createdAt)],
         with: {
           operations: true,
@@ -148,7 +148,7 @@ replicate.command('run', {
         where: and(
           eq(schema.repairs.id, c.args.replicateId),
           inArray(schema.repairs.status, ['pending', 'failed']),
-          isNotNull(schema.repairs.repairDataSetId)
+          eq(schema.repairs.kind, 'replicate')
         ),
       })
       if (!repair) {

package/src/commands/second-copy.ts

@@ -0,0 +1,187 @@
+import { and, desc, eq, inArray } from 'drizzle-orm'
+import { Cli, z } from 'incur'
+import { isAddress } from 'viem'
+import { repairDelete } from '../db/repair-delete.ts'
+import { secondCopyCreate } from '../db/second-copy-create.ts'
+import { contextMiddleware, contextSchema } from '../middleware.ts'
+import { runAddPieces } from '../pipeline/add-pieces.ts'
+import { ensureRepairDataset } from '../pipeline/create-datasets.ts'
+import { globalOptions } from '../utils.ts'
+
+export const secondCopy = Cli.create('second-copy', {
+  description: 'Second-copy piece commands',
+  vars: contextSchema,
+})
+
+secondCopy.command('create', {
+  description: 'Create a second-copy job from a pieces file',
+  options: globalOptions.extend({
+    piecesFile: z.string().describe('Path to JSON file with aggregates[].piece_cid'),
+    sourceProviderUrl: z.string().url().describe('Source provider URL to pull pieces from'),
+    targetProviderId: z.coerce.bigint().describe('Target provider ID for second-copy'),
+  }),
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const { piecesFile, sourceProviderUrl, targetProviderId } = c.options
+
+      const secondCopyId = await secondCopyCreate({
+        ...c.var,
+        piecesFile,
+        sourceProviderUrl,
+        targetProviderId,
+      })
+
+      return c.ok({
+        secondCopyId,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to create the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('list', {
+  description: 'List all second-copy jobs',
+  options: globalOptions,
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const localSchema = c.var.localDb._.fullSchema
+      const jobs = await c.var.localDb.query.repairs.findMany({
+        where: eq(localSchema.repairs.kind, 'second_copy'),
+        orderBy: [desc(localSchema.repairs.createdAt)],
+        with: {
+          operations: true,
+        },
+      })
+
+      const secondCopies = jobs.map((repair) => {
+        const { operations, ...job } = repair
+        return {
+          id: job.id,
+          status: job.status,
+          sourceProviderUrl: job.sourceProviderUrl,
+          targetProviderId: job.targetProviderId,
+          targetProviderUrl: job.targetProviderUrl,
+          targetDataSetId: job.targetDataSetId,
+          blockNumber: job.blockNumber,
+          operations: operations.length,
+          pending: operations.filter((operation) => operation.status === 'pending').length,
+          failed: operations.filter((operation) => operation.status === 'failed').length,
+          completed: operations.filter((operation) => operation.status === 'completed').length,
+          skipped: operations.filter((operation) => operation.status === 'skipped').length,
+        }
+      })
+
+      return c.ok({
+        secondCopies,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to list second-copy jobs',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('delete', {
+  description: 'Delete a second-copy job',
+  args: z.object({
+    secondCopyId: z.coerce.number().describe('Second-copy ID to delete'),
+  }),
+  options: globalOptions,
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const { deleted, operationsDeleted } = await repairDelete({
+        localDb: c.var.localDb,
+        repairId: c.args.secondCopyId,
+      })
+
+      if (!deleted) {
+        return c.error({
+          code: 'SECOND_COPY_NOT_FOUND',
+          message: 'Second-copy job not found',
+          retryable: false,
+        })
+      }
+
+      return c.ok({
+        secondCopyId: c.args.secondCopyId,
+        operationsDeleted,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to delete the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})
+
+secondCopy.command('run', {
+  description: 'Run a second-copy job',
+  args: z.object({
+    secondCopyId: z.coerce.number().describe('Second-copy ID to run'),
+  }),
+  options: globalOptions.extend({
+    concurrency: z.coerce.number().min(1).max(10).default(4).describe('Concurrency level'),
+    batchSize: z.coerce.number().min(1).max(40).default(40).describe('Max pieces per batch'),
+    payer: z.string().refine(isAddress, 'Invalid address').optional().describe('Payer address'),
+  }),
+  middleware: [contextMiddleware],
+  run: async (c) => {
+    try {
+      const schema = c.var.localDb._.fullSchema
+      const payer = c.options.payer ?? c.var.client.account.address
+      const repair = await c.var.localDb.query.repairs.findFirst({
+        where: and(
+          eq(schema.repairs.id, c.args.secondCopyId),
+          inArray(schema.repairs.status, ['pending', 'failed']),
+          eq(schema.repairs.kind, 'second_copy')
+        ),
+      })
+      if (!repair) {
+        return c.error({
+          code: 'SECOND_COPY_NOT_FOUND',
+          message: 'Second-copy job not found, it may have already been run or completed',
+          retryable: false,
+        })
+      }
+
+      await ensureRepairDataset({
+        ...c.var,
+        repair,
+        payer,
+      })
+
+      await runAddPieces({
+        ...c.var,
+        repair,
+        concurrency: c.options.concurrency,
+        batchSize: c.options.batchSize,
+      })
+      return c.ok({
+        secondCopyId: repair.id,
+      })
+    } catch (error) {
+      console.error(error)
+      return c.error({
+        code: 'SECOND_COPY_FAILED',
+        message: error instanceof Error ? error.message : 'Failed to run the second-copy job',
+        retryable: true,
+      })
+    }
+  },
+})

package/src/commands/setup.ts

@@ -3,21 +3,16 @@ import { Cli, z } from 'incur'
 import path from 'path'
 import type { Hash } from 'viem'
 import { privateKeyToAccount } from 'viem/accounts'
+import { DEFAULT_API_URL } from '../api.ts'
 import { config, createLocalDatabase, globalOptions, migrateLocalDatabase } from '../utils.ts'
 
-function validatePostgresUrl(value: string) {
-  let url: URL
+function validateHttpUrl(value: string) {
   try {
-    url = new URL(value)
+    const url = new URL(value)
+    return url.protocol === 'http:' || url.protocol === 'https:'
   } catch {
     return false
   }
-
-  if (url.protocol !== 'postgresql:' && url.protocol !== 'postgres:') {
-    return false
-  }
-
-  return Boolean(url.hostname && url.username && url.password && url.pathname.length > 1)
 }
 
 export const setup = Cli.create('setup', {
@@ -44,35 +39,18 @@ export const setup = Cli.create('setup', {
         })
       }
 
-      // Indexer URL
-      const indexerMainnetUrl = await p.text({
-        message: 'Enter your Mainnet Indexer Postgres URL',
-        validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
-          }
-        },
-        initialValue: config.get('indexerMainnetUrl'),
-        withGuide: false,
-      })
-      if (p.isCancel(indexerMainnetUrl)) {
-        return c.error({
-          code: 'SETUP_CANCELLED',
-          message: 'Setup cancelled',
-          retryable: false,
-        })
-      }
-      const indexerCalibrationUrl = await p.text({
-        message: 'Enter your Calibration Indexer Postgres URL',
+      // API URL
+      const apiUrl = await p.text({
+        message: 'Enter the Filecoin API URL',
         validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
+          if (!value || !validateHttpUrl(value)) {
+            return `Invalid http(s) URL!`
           }
         },
-        initialValue: config.get('indexerCalibrationUrl'),
+        initialValue: config.get('apiUrl') ?? DEFAULT_API_URL,
         withGuide: false,
       })
-      if (p.isCancel(indexerCalibrationUrl)) {
+      if (p.isCancel(apiUrl)) {
         return c.error({
           code: 'SETUP_CANCELLED',
           message: 'Setup cancelled',
@@ -129,8 +107,7 @@ export const setup = Cli.create('setup', {
 
       // Set config
       config.set('privateKey', pk)
-      config.set('indexerMainnetUrl', indexerMainnetUrl)
-      config.set('indexerCalibrationUrl', indexerCalibrationUrl)
+      config.set('apiUrl', apiUrl)
       config.set('chainId', chainId)
       config.set('dbPath', dbPath)
       config.set('source', source)

package/src/db/dedupe-cids.ts

@@ -1,37 +1,20 @@
-import { and, eq, inArray } from 'drizzle-orm'
 import type { OperationSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase } from '../types.ts'
+import type { LocalDatabase } from '../types.ts'
 import { upsertOperations } from './upsert-operations.ts'
 
-export type SyncPiecesOnchainOptions = {
-  indexerDb: IndexerDatabase
+export type DedupeCidsOptions = {
   localDb: LocalDatabase
-  dataSetId: bigint
+  /** CIDs already in the target repair dataset. */
+  existingCids: ReadonlySet<string>
   operations: OperationSelect[]
 }
 
 /**
- * Dedupe operations by CID on the target repair dataset.
+ * Mark operations whose CID is already in the target repair dataset as completed.
  * Returns operations that are not onchain.
  */
-export async function dedupeCids({ indexerDb, localDb, dataSetId, operations }: SyncPiecesOnchainOptions) {
-  const cids = operations.map((operation) => operation.cid)
-  const schema = indexerDb._.fullSchema
-
-  const rows = await indexerDb
-    .select({ cid: schema.pieces.cid })
-    .from(schema.pieces)
-    .where(
-      and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false), inArray(schema.pieces.cid, cids))
-    )
-
-  const existingCids = new Set<string>()
-  const completedOperations: OperationSelect[] = []
-
-  for (const row of rows) {
-    completedOperations.push(...operations.filter((operation) => operation.cid === row.cid))
-    existingCids.add(row.cid)
-  }
+export async function dedupeCids({ localDb, existingCids, operations }: DedupeCidsOptions) {
+  const completedOperations = operations.filter((operation) => existingCids.has(operation.cid))
 
   if (completedOperations.length > 0) {
     await upsertOperations({

package/src/db/find-repair-dataset.ts

@@ -1,49 +1,38 @@
-import { and, asc, eq, isNull } from 'drizzle-orm'
 import type { Address } from 'viem'
-import type { IndexerDatabase } from '../types.ts'
+import { listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
 
 export type FindRepairDatasetOptions = {
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   providerId: bigint
   payer: Address
   source: string
 }
 
 /**
- * Find one IPFS-enabled repair dataset for a provider at the repair block, if it exists.
+ * Find one live IPFS-enabled repair dataset for a provider, if it exists.
  *
  * When multiple datasets match, the lowest `dataSetId` is returned.
  */
 export async function findRepairDataset({
-  indexerDb,
+  api,
   providerId,
   payer,
   source,
 }: FindRepairDatasetOptions): Promise<bigint | null> {
-  const schema = indexerDb._.fullSchema
+  const dataSets = await listDataSets(api, {
+    owner: payer.toLowerCase(),
+    providerId,
+    deleted: false,
+    withCdn: false,
+  })
 
-  const result = await indexerDb
-    .select({
-      dataSetId: schema.dataSets.dataSetId,
-    })
-    .from(schema.dataSets)
-    .where(
-      and(
-        eq(schema.dataSets.providerId, providerId),
-        eq(schema.dataSets.deleted, false),
-        isNull(schema.dataSets.pdpEndEpoch),
-        eq(schema.dataSets.payer, payer.toLowerCase()),
-        eq(schema.dataSets.source, source),
-        eq(schema.dataSets.withCdn, false),
-        eq(schema.dataSets.withIpfsIndexing, true)
-      )
-    )
-    .orderBy(asc(schema.dataSets.dataSetId))
-    .limit(1)
-
-  if (result.length === 0) {
-    return null
+  let dataSetId: bigint | null = null
+  for (const dataSet of dataSets) {
+    if (dataSet.pdpEndEpoch != null || dataSet.source !== source || !dataSet.withIpfsIndexing) continue
+    const id = BigInt(dataSet.dataSetId)
+    if (dataSetId == null || id < dataSetId) dataSetId = id
   }
 
-  return result[0].dataSetId
+  return dataSetId
 }

package/src/db/get-active-data-set-ids.ts

@@ -0,0 +1,29 @@
+import { type DataSet, listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
+
+/**
+ * Whether a data set's pieces count at the repair block: not deleted, and either not terminated
+ * (no PDP end epoch) or terminated with an end epoch after `blockNumber`, so the provider still
+ * has to prove it.
+ */
+export function isDataSetActiveAt(dataSet: Pick<DataSet, 'deleted' | 'pdpEndEpoch'>, blockNumber: bigint): boolean {
+  return !dataSet.deleted && (dataSet.pdpEndEpoch == null || BigInt(dataSet.pdpEndEpoch) > blockNumber)
+}
+
+/**
+ * IDs of a provider's data sets that are active at the repair block (see {@link isDataSetActiveAt}).
+ */
+export async function getActiveDataSetIds({
+  api,
+  providerId,
+  blockNumber,
+}: {
+  api: IndexerApi
+  providerId: bigint
+  blockNumber: bigint
+}): Promise<Set<string>> {
+  const dataSets = await listDataSets(api, { providerId, deleted: false })
+  return new Set(
+    dataSets.filter((dataSet) => isDataSetActiveAt(dataSet, blockNumber)).map((dataSet) => dataSet.dataSetId)
+  )
+}