@filoz/repair-cli 0.4.0 → 1.0.0

package/src/api.ts

@@ -0,0 +1,220 @@
+import {
+  ApiError,
+  createApiClient,
+  type DataSet,
+  DEFAULT_API_URL,
+  type Piece,
+  type PieceWithDataSet,
+  type Provider,
+  unwrap,
+} from '@filecoin-project/api-client'
+import { pipeline, withRateLimit, withRetry, withTimeout } from 'fetch-extras'
+import type { IndexerApi } from './types.ts'
+
+export { ApiError, DEFAULT_API_URL } from '@filecoin-project/api-client'
+export type { DataSet, Piece, PieceWithDataSet, Provider }
+
+/** filecoin-api network path segment. */
+export type Network = 'mainnet' | 'calibration'
+
+/** Largest page size filecoin-api accepts. */
+export const MAX_PAGE_SIZE = 200
+
+/** Retries for network errors and `408`, `429` and `5xx` responses. */
+const MAX_RETRIES = 5
+
+/** Per-attempt request timeout; a timed-out request fails instead of retrying. */
+const REQUEST_TIMEOUT_MS = 30_000
+
+/** filecoin-api allows 120 requests per minute per client IP. */
+const RATE_LIMIT = { requestsPerInterval: 120, interval: 60_000 }
+
+/** Map a chain ID to its filecoin-api network. */
+export function networkFromChainId(chainId: number): Network {
+  return chainId === 314 ? 'mainnet' : 'calibration'
+}
+
+/**
+ * Create a filecoin-api client for a chain.
+ *
+ * @param options.baseUrl - API origin; defaults to the client's production URL.
+ * @param options.chainId - Chain ID used to pick the network.
+ */
+export function createIndexerApi({ baseUrl, chainId }: { baseUrl?: string; chainId: number }): IndexerApi {
+  // Pace requests under the API's limit, and still retry 429s (other clients may share the IP),
+  // honouring Retry-After.
+  const fetchWithRetry = pipeline(
+    fetch,
+    withTimeout(REQUEST_TIMEOUT_MS),
+    withRateLimit(RATE_LIMIT),
+    withRetry({ retries: MAX_RETRIES })
+  )
+  return {
+    client: createApiClient({ baseUrl: baseUrl ?? DEFAULT_API_URL, fetch: fetchWithRetry }),
+    network: networkFromChainId(chainId),
+  }
+}
+
+/** Return `null` instead of throwing for a filecoin-api 404. */
+async function orNull<T>(promise: Promise<T>): Promise<T | null> {
+  try {
+    return await promise
+  } catch (error) {
+    if (ApiError.isNotFound(error)) return null
+    throw error
+  }
+}
+
+/** Collect every item from a cursor-paginated walk. */
+async function collect<T>(pages: AsyncIterable<T[]>): Promise<T[]> {
+  const items: T[] = []
+  for await (const page of pages) items.push(...page)
+  return items
+}
+
+/**
+ * Walk a cursor-paginated list, yielding one page of items at a time.
+ *
+ * @param fetchPage - Fetches the page after `cursor` (`undefined` for the first page).
+ */
+async function* paginate<T>(
+  fetchPage: (cursor: string | undefined) => Promise<{ data: T[]; nextCursor: string | null }>
+): AsyncGenerator<T[]> {
+  let cursor: string | undefined
+  do {
+    const page = await fetchPage(cursor)
+    yield page.data
+    cursor = page.nextCursor ?? undefined
+  } while (cursor)
+}
+
+/** Get a provider by ID, or `null` when it does not exist. */
+export function getProvider({ client, network }: IndexerApi, providerId: bigint): Promise<Provider | null> {
+  return orNull(
+    client
+      .GET('/{network}/providers/{providerId}', {
+        params: { path: { network, providerId: providerId.toString() } },
+      })
+      .then((result) => unwrap(result).data)
+  )
+}
+
+/** List every provider matching the filters. */
+export function listProviders(
+  { client, network }: IndexerApi,
+  query: { active?: boolean; approved?: boolean; endorsed?: boolean } = {}
+): Promise<Provider[]> {
+  return collect(
+    paginate((cursor) =>
+      client
+        .GET('/{network}/providers', {
+          params: {
+            path: { network },
+            query: {
+              limit: MAX_PAGE_SIZE,
+              cursor,
+              active: bool(query.active),
+              approved: bool(query.approved),
+              endorsed: bool(query.endorsed),
+            },
+          },
+        })
+        .then(unwrap)
+    )
+  )
+}
+
+/** Get a data set by ID, or `null` when it does not exist. */
+export function getDataSet({ client, network }: IndexerApi, dataSetId: bigint): Promise<DataSet | null> {
+  return orNull(
+    client
+      .GET('/{network}/data-sets/{dataSetId}', {
+        params: { path: { network, dataSetId: dataSetId.toString() } },
+      })
+      .then((result) => unwrap(result).data)
+  )
+}
+
+/** List every data set matching the filters, newest ID first. */
+export function listDataSets(
+  { client, network }: IndexerApi,
+  query: { owner?: string; providerId?: bigint; deleted?: boolean; withCdn?: boolean }
+): Promise<DataSet[]> {
+  return collect(
+    paginate((cursor) =>
+      client
+        .GET('/{network}/data-sets', {
+          params: {
+            path: { network },
+            query: {
+              limit: MAX_PAGE_SIZE,
+              cursor,
+              owner: query.owner,
+              provider_id: query.providerId?.toString(),
+              deleted: bool(query.deleted),
+              with_cdn: bool(query.withCdn),
+            },
+          },
+        })
+        .then(unwrap)
+    )
+  )
+}
+
+/** Walk the pieces of one data set page by page, newest piece ID first. */
+export function dataSetPiecePages(
+  { client, network }: IndexerApi,
+  dataSetId: bigint,
+  query: { removed?: boolean } = {}
+): AsyncGenerator<Piece[]> {
+  return paginate((cursor) =>
+    client
+      .GET('/{network}/data-sets/{dataSetId}/pieces', {
+        params: {
+          path: { network, dataSetId: dataSetId.toString() },
+          query: { limit: MAX_PAGE_SIZE, cursor, removed: bool(query.removed) },
+        },
+      })
+      .then(unwrap)
+  )
+}
+
+/** List every piece in one data set, newest piece ID first. */
+export function listDataSetPieces(
+  api: IndexerApi,
+  dataSetId: bigint,
+  query: { removed?: boolean } = {}
+): Promise<Piece[]> {
+  return collect(dataSetPiecePages(api, dataSetId, query))
+}
+
+/**
+ * Walk the pieces of every data set a provider stores, page by page, newest data set and piece
+ * first. One walk covers all of the provider's data sets.
+ */
+export function providerPiecePages(
+  { client, network }: IndexerApi,
+  providerId: bigint,
+  query: { removed?: boolean } = {}
+): AsyncGenerator<PieceWithDataSet[]> {
+  return paginate((cursor) =>
+    client
+      .GET('/{network}/pieces', {
+        params: {
+          path: { network },
+          query: { limit: MAX_PAGE_SIZE, cursor, provider_id: providerId.toString(), removed: bool(query.removed) },
+        },
+      })
+      .then(unwrap)
+  )
+}
+
+/** Encode an optional boolean filter as a query string value. */
+function bool(value: boolean | undefined): 'true' | 'false' | undefined {
+  return value === undefined ? undefined : value ? 'true' : 'false'
+}
+
+/** Convert API piece metadata to the string record stored on local operations. */
+export function toStringMetadata(metadata: Record<string, unknown> | null): Record<string, string> {
+  return Object.fromEntries(Object.entries(metadata ?? {}).map(([key, value]) => [key, String(value)]))
+}

package/src/commands/datasets.ts

@@ -1,9 +1,9 @@
 import * as p from '@clack/prompts'
 import * as SP from '@filoz/synapse-core/sp'
 import { getPdpDataSet } from '@filoz/synapse-core/warm-storage'
-import { and, asc, eq } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { isAddress } from 'viem'
+import { getDataSet, getProvider, listDataSetPieces, listDataSets, listProviders } from '../api.ts'
 import { contextMiddleware, contextSchema } from '../middleware.ts'
 import { globalOptions, hashLink } from '../utils.ts'
 
@@ -18,42 +18,33 @@ datasets.command('list', {
   options: globalOptions.extend({
     providerId: z.coerce.bigint().optional().describe('Filter datasets by provider ID'),
     payer: z.string().refine(isAddress, 'Invalid address').optional().describe('Filter datasets by payer address'),
+    stats: z.boolean().optional().default(false).describe('Include piece counts (slow: walks every piece)'),
   }),
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const schema = c.var.indexerDb._.fullSchema
+      const { api } = c.var
       const payer = c.options.payer?.toLowerCase() ?? c.var.client.account.address.toLowerCase()
-      const conditions = [eq(schema.dataSets.deleted, false), eq(schema.dataSets.payer, payer)]
+      const [rows, providers] = await Promise.all([
+        listDataSets(api, { owner: payer, providerId: c.options.providerId, deleted: false }),
+        listProviders(api),
+      ])
+      const serviceUrls = new Map(providers.map((provider) => [provider.providerId, provider.serviceUrl]))
 
-      if (c.options.providerId != null) {
-        conditions.push(eq(schema.dataSets.providerId, c.options.providerId))
-      }
-
-      const datasets = await c.var.indexerDb.query.dataSets.findMany({
-        where: and(...conditions),
-        with: {
-          provider: true,
-          pieces: {
-            where: eq(c.var.indexerDb._.fullSchema.pieces.removed, false),
-          },
-        },
-        orderBy: [asc(schema.dataSets.dataSetId)],
-      })
-
-      const datasetsFlattened = datasets.map((dataset) => {
-        const { provider, pieces } = dataset
-        return {
+      const datasetsFlattened = []
+      for (const dataset of rows.reverse()) {
+        datasetsFlattened.push({
           id: dataset.dataSetId,
           withCdn: dataset.withCdn,
           withIpfsIndexing: dataset.withIpfsIndexing,
           source: dataset.source,
-          provider: provider.serviceUrl,
+          provider: dataset.providerId == null ? null : serviceUrls.get(dataset.providerId),
           pdpEndEpoch: dataset.pdpEndEpoch,
-          pieces: pieces.length,
-          //   metadata: JSON.stringify(dataset.metadata),
-        }
-      })
+          ...(c.options.stats && {
+            pieces: (await listDataSetPieces(api, BigInt(dataset.dataSetId), { removed: false })).length,
+          }),
+        })
+      }
 
       return c.ok({
         datasets: datasetsFlattened,
@@ -138,15 +129,8 @@ datasets.command('show', {
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const dataset = await c.var.indexerDb.query.dataSets.findFirst({
-        where: eq(c.var.indexerDb._.fullSchema.dataSets.dataSetId, c.args.id),
-        with: {
-          provider: true,
-          pieces: {
-            where: eq(c.var.indexerDb._.fullSchema.pieces.removed, false),
-          },
-        },
-      })
+      const { api } = c.var
+      const dataset = await getDataSet(api, c.args.id)
 
       if (!dataset) {
         return c.error({
@@ -155,7 +139,11 @@ datasets.command('show', {
           retryable: false,
         })
       }
-      return c.ok(dataset)
+      const [provider, pieces] = await Promise.all([
+        dataset.providerId == null ? null : getProvider(api, BigInt(dataset.providerId)),
+        listDataSetPieces(api, c.args.id, { removed: false }),
+      ])
+      return c.ok({ ...dataset, provider, pieces: pieces.reverse() })
     } catch (error) {
       if (c.options.debug) {
         console.error(error)

package/src/commands/providers.ts

@@ -1,6 +1,7 @@
-import { and, asc, count, eq, inArray, isNull, lte, or, type SQLWrapper, sum } from 'drizzle-orm'
 import { Cli, z } from 'incur'
 import { getBlockNumber } from 'viem/actions'
+import { listProviders, providerPiecePages } from '../api.ts'
+import { getActiveDataSetIds } from '../db/get-active-data-set-ids.ts'
 import { contextMiddleware, contextSchema } from '../middleware.ts'
 import { globalOptions } from '../utils.ts'
 
@@ -12,6 +13,13 @@ function formatBytesAsGb(bytes: bigint): string {
   return `${whole}.${fraction.toString().padStart(2, '0')} GB`
 }
 
+/** Compare decimal ID strings numerically. */
+function compareIds(a: string, b: string): number {
+  const x = BigInt(a)
+  const y = BigInt(b)
+  return x < y ? -1 : x > y ? 1 : 0
+}
+
 export const providers = Cli.create('providers', {
   description: 'Provider commands',
   options: globalOptions,
@@ -22,51 +30,36 @@ providers.command('list', {
   description: 'List all providers from the indexer',
   options: globalOptions.extend({
     all: z.boolean().optional().default(false).describe('Include all providers'),
+    stats: z
+      .boolean()
+      .optional()
+      .default(false)
+      .describe('Include piece count and total size (slow: walks every piece of each provider)'),
   }),
   middleware: [contextMiddleware],
   run: async (c) => {
     try {
-      const schema = c.var.indexerDb._.fullSchema
-      const filters: (SQLWrapper | undefined)[] = [
-        eq(schema.providers.providerActive, true),
-        eq(schema.providers.pdpProductActive, true),
-      ]
-      if (!c.options.all) {
-        filters.push(or(eq(schema.providers.approved, true), eq(schema.providers.endorsed, true)))
-      }
-      const blockNumber = await getBlockNumber(c.var.client)
-      const rows = await c.var.indexerDb.query.providers.findMany({
-        orderBy: [asc(schema.providers.providerId)],
-        where: and(...filters),
-      })
-
-      const providerIds = rows.map((provider) => provider.providerId)
-      const statsByProviderId = new Map<bigint, { pieceCount: number; totalSize: bigint }>()
+      const { api } = c.var
+      const rows = (await listProviders(api, { active: true }))
+        .filter((provider) => provider.pdpProductActive && (c.options.all || provider.approved || provider.endorsed))
+        .sort((a, b) => compareIds(a.providerId, b.providerId))
 
-      if (providerIds.length > 0) {
-        const stats = await c.var.indexerDb
-          .select({
-            providerId: schema.dataSets.providerId,
-            pieceCount: count(schema.pieces.pieceId),
-            totalSize: sum(schema.pieces.rawSize),
-          })
-          .from(schema.pieces)
-          .innerJoin(schema.dataSets, eq(schema.pieces.dataSetId, schema.dataSets.dataSetId))
-          .where(
-            and(
-              inArray(schema.dataSets.providerId, providerIds),
-              eq(schema.dataSets.deleted, false),
-              or(isNull(schema.dataSets.pdpEndEpoch), lte(schema.dataSets.pdpEndEpoch, blockNumber)),
-              eq(schema.pieces.removed, false)
-            )
-          )
-          .groupBy(schema.dataSets.providerId)
+      const statsByProviderId = new Map<string, { pieceCount: number; totalSize: bigint }>()
 
-        for (const stat of stats) {
-          statsByProviderId.set(stat.providerId, {
-            pieceCount: stat.pieceCount,
-            totalSize: stat.totalSize == null ? 0n : BigInt(stat.totalSize),
-          })
+      if (c.options.stats && rows.length > 0) {
+        const blockNumber = await getBlockNumber(c.var.client)
+        for (const provider of rows) {
+          const providerId = BigInt(provider.providerId)
+          const activeDataSetIds = await getActiveDataSetIds({ api, providerId, blockNumber })
+          const stats = { pieceCount: 0, totalSize: 0n }
+          for await (const pieces of providerPiecePages(api, providerId, { removed: false })) {
+            for (const piece of pieces) {
+              if (!activeDataSetIds.has(piece.dataSetId)) continue
+              stats.pieceCount++
+              stats.totalSize += BigInt(piece.rawSize ?? 0)
+            }
+          }
+          statsByProviderId.set(provider.providerId, stats)
         }
       }
 
@@ -78,8 +71,7 @@ providers.command('list', {
           serviceUrl: provider.serviceUrl,
           approved: provider.approved,
           endorsed: provider.endorsed,
-          pieceCount: stats?.pieceCount ?? 0,
-          totalSize: formatBytesAsGb(stats?.totalSize ?? 0n),
+          ...(stats && { pieceCount: stats.pieceCount, totalSize: formatBytesAsGb(stats.totalSize) }),
         }
       })
 

package/src/commands/setup.ts

@@ -3,21 +3,16 @@ import { Cli, z } from 'incur'
 import path from 'path'
 import type { Hash } from 'viem'
 import { privateKeyToAccount } from 'viem/accounts'
+import { DEFAULT_API_URL } from '../api.ts'
 import { config, createLocalDatabase, globalOptions, migrateLocalDatabase } from '../utils.ts'
 
-function validatePostgresUrl(value: string) {
-  let url: URL
+function validateHttpUrl(value: string) {
   try {
-    url = new URL(value)
+    const url = new URL(value)
+    return url.protocol === 'http:' || url.protocol === 'https:'
   } catch {
     return false
   }
-
-  if (url.protocol !== 'postgresql:' && url.protocol !== 'postgres:') {
-    return false
-  }
-
-  return Boolean(url.hostname && url.username && url.password && url.pathname.length > 1)
 }
 
 export const setup = Cli.create('setup', {
@@ -44,35 +39,18 @@ export const setup = Cli.create('setup', {
         })
       }
 
-      // Indexer URL
-      const indexerMainnetUrl = await p.text({
-        message: 'Enter your Mainnet Indexer Postgres URL',
-        validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
-          }
-        },
-        initialValue: config.get('indexerMainnetUrl'),
-        withGuide: false,
-      })
-      if (p.isCancel(indexerMainnetUrl)) {
-        return c.error({
-          code: 'SETUP_CANCELLED',
-          message: 'Setup cancelled',
-          retryable: false,
-        })
-      }
-      const indexerCalibrationUrl = await p.text({
-        message: 'Enter your Calibration Indexer Postgres URL',
+      // API URL
+      const apiUrl = await p.text({
+        message: 'Enter the Filecoin API URL',
         validate(value) {
-          if (!value || !validatePostgresUrl(value)) {
-            return `Invalid postgres URL!`
+          if (!value || !validateHttpUrl(value)) {
+            return `Invalid http(s) URL!`
           }
         },
-        initialValue: config.get('indexerCalibrationUrl'),
+        initialValue: config.get('apiUrl') ?? DEFAULT_API_URL,
         withGuide: false,
       })
-      if (p.isCancel(indexerCalibrationUrl)) {
+      if (p.isCancel(apiUrl)) {
         return c.error({
           code: 'SETUP_CANCELLED',
           message: 'Setup cancelled',
@@ -129,8 +107,7 @@ export const setup = Cli.create('setup', {
 
       // Set config
       config.set('privateKey', pk)
-      config.set('indexerMainnetUrl', indexerMainnetUrl)
-      config.set('indexerCalibrationUrl', indexerCalibrationUrl)
+      config.set('apiUrl', apiUrl)
       config.set('chainId', chainId)
       config.set('dbPath', dbPath)
       config.set('source', source)

package/src/db/dedupe-cids.ts

@@ -1,37 +1,20 @@
-import { and, eq, inArray } from 'drizzle-orm'
 import type { OperationSelect } from '../local-schema.ts'
-import type { IndexerDatabase, LocalDatabase } from '../types.ts'
+import type { LocalDatabase } from '../types.ts'
 import { upsertOperations } from './upsert-operations.ts'
 
-export type SyncPiecesOnchainOptions = {
-  indexerDb: IndexerDatabase
+export type DedupeCidsOptions = {
   localDb: LocalDatabase
-  dataSetId: bigint
+  /** CIDs already in the target repair dataset. */
+  existingCids: ReadonlySet<string>
   operations: OperationSelect[]
 }
 
 /**
- * Dedupe operations by CID on the target repair dataset.
+ * Mark operations whose CID is already in the target repair dataset as completed.
  * Returns operations that are not onchain.
  */
-export async function dedupeCids({ indexerDb, localDb, dataSetId, operations }: SyncPiecesOnchainOptions) {
-  const cids = operations.map((operation) => operation.cid)
-  const schema = indexerDb._.fullSchema
-
-  const rows = await indexerDb
-    .select({ cid: schema.pieces.cid })
-    .from(schema.pieces)
-    .where(
-      and(eq(schema.pieces.dataSetId, dataSetId), eq(schema.pieces.removed, false), inArray(schema.pieces.cid, cids))
-    )
-
-  const existingCids = new Set<string>()
-  const completedOperations: OperationSelect[] = []
-
-  for (const row of rows) {
-    completedOperations.push(...operations.filter((operation) => operation.cid === row.cid))
-    existingCids.add(row.cid)
-  }
+export async function dedupeCids({ localDb, existingCids, operations }: DedupeCidsOptions) {
+  const completedOperations = operations.filter((operation) => existingCids.has(operation.cid))
 
   if (completedOperations.length > 0) {
     await upsertOperations({

package/src/db/find-repair-dataset.ts

@@ -1,49 +1,38 @@
-import { and, asc, eq, isNull } from 'drizzle-orm'
 import type { Address } from 'viem'
-import type { IndexerDatabase } from '../types.ts'
+import { listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
 
 export type FindRepairDatasetOptions = {
-  indexerDb: IndexerDatabase
+  api: IndexerApi
   providerId: bigint
   payer: Address
   source: string
 }
 
 /**
- * Find one IPFS-enabled repair dataset for a provider at the repair block, if it exists.
+ * Find one live IPFS-enabled repair dataset for a provider, if it exists.
  *
  * When multiple datasets match, the lowest `dataSetId` is returned.
  */
 export async function findRepairDataset({
-  indexerDb,
+  api,
   providerId,
   payer,
   source,
 }: FindRepairDatasetOptions): Promise<bigint | null> {
-  const schema = indexerDb._.fullSchema
+  const dataSets = await listDataSets(api, {
+    owner: payer.toLowerCase(),
+    providerId,
+    deleted: false,
+    withCdn: false,
+  })
 
-  const result = await indexerDb
-    .select({
-      dataSetId: schema.dataSets.dataSetId,
-    })
-    .from(schema.dataSets)
-    .where(
-      and(
-        eq(schema.dataSets.providerId, providerId),
-        eq(schema.dataSets.deleted, false),
-        isNull(schema.dataSets.pdpEndEpoch),
-        eq(schema.dataSets.payer, payer.toLowerCase()),
-        eq(schema.dataSets.source, source),
-        eq(schema.dataSets.withCdn, false),
-        eq(schema.dataSets.withIpfsIndexing, true)
-      )
-    )
-    .orderBy(asc(schema.dataSets.dataSetId))
-    .limit(1)
-
-  if (result.length === 0) {
-    return null
+  let dataSetId: bigint | null = null
+  for (const dataSet of dataSets) {
+    if (dataSet.pdpEndEpoch != null || dataSet.source !== source || !dataSet.withIpfsIndexing) continue
+    const id = BigInt(dataSet.dataSetId)
+    if (dataSetId == null || id < dataSetId) dataSetId = id
   }
 
-  return result[0].dataSetId
+  return dataSetId
 }

package/src/db/get-active-data-set-ids.ts

@@ -0,0 +1,29 @@
+import { type DataSet, listDataSets } from '../api.ts'
+import type { IndexerApi } from '../types.ts'
+
+/**
+ * Whether a data set's pieces count at the repair block: not deleted, and either not terminated
+ * (no PDP end epoch) or terminated with an end epoch after `blockNumber`, so the provider still
+ * has to prove it.
+ */
+export function isDataSetActiveAt(dataSet: Pick<DataSet, 'deleted' | 'pdpEndEpoch'>, blockNumber: bigint): boolean {
+  return !dataSet.deleted && (dataSet.pdpEndEpoch == null || BigInt(dataSet.pdpEndEpoch) > blockNumber)
+}
+
+/**
+ * IDs of a provider's data sets that are active at the repair block (see {@link isDataSetActiveAt}).
+ */
+export async function getActiveDataSetIds({
+  api,
+  providerId,
+  blockNumber,
+}: {
+  api: IndexerApi
+  providerId: bigint
+  blockNumber: bigint
+}): Promise<Set<string>> {
+  const dataSets = await listDataSets(api, { providerId, deleted: false })
+  return new Set(
+    dataSets.filter((dataSet) => isDataSetActiveAt(dataSet, blockNumber)).map((dataSet) => dataSet.dataSetId)
+  )
+}