@secondlayer/shared 7.6.0 → 7.7.0

package/dist/src/archive/copy-loader.d.ts

@@ -0,0 +1,49 @@
+import { Writable } from "node:stream";
+/**
+* Streaming `COPY ... FROM STDIN` loader for canonical archive rows.
+*
+* Restoring history row-by-row is not viable at chain scale: batched INSERTs
+* measured ~2k rows/s on this schema, which is days for a full genesis restore.
+* COPY moves the same data at ~64k rows/s because it is one statement per
+* partition with no per-row round-trip.
+*
+* This module deliberately knows nothing about Parquet or about where rows come
+* from — it accepts the archive's row shape as an async iterable. That keeps the
+* heavy file-format dependency out of every consumer that only needs to write
+* rows, and lets the CLI and the indexer share one loader rather than growing
+* two subtly different CSV encoders.
+*/
+type ArchiveDataset = "blocks" | "transactions" | "events";
+/** Row shapes exactly as the archive's Parquet columns are named. */
+type ArchiveRow = Record<string, unknown>;
+/**
+* CSV encoding for `WITH (FORMAT csv, NULL '\N')`: every present value is
+* quoted with embedded quotes doubled, so a value that happens to equal the
+* NULL marker cannot be misread as SQL NULL. Only a genuinely absent value is
+* written unquoted as `\N`.
+*/
+declare function csvField(value: string | number | null | undefined): string;
+declare function toCsvLine(dataset: ArchiveDataset, row: ArchiveRow): string;
+/**
+* `canonical` is omitted from the blocks column list: every archived block is
+* canonical by definition, and the schema defaults it to true, so carrying it
+* through every row would be dead weight.
+*/
+declare function copyStatement(dataset: ArchiveDataset): string;
+/**
+* Stream rows onto an open COPY writable. The caller owns opening it (drivers
+* differ in how a COPY stream is obtained) and this owns the wire format,
+* backpressure, and failure semantics.
+*
+* On error the stream is destroyed rather than ended, so Postgres aborts the
+* COPY instead of committing a partial one — an interrupted load must leave
+* nothing behind, not half a partition.
+*/
+declare function writeRowsToCopyStream(params: {
+	writable: Writable
+	dataset: ArchiveDataset
+	rows: AsyncIterable<ArchiveRow>
+	onProgress?: (rowsWritten: number) => void
+	progressEvery?: number
+}): Promise<number>;
+export { writeRowsToCopyStream, toCsvLine, csvField, copyStatement, ArchiveRow, ArchiveDataset };

package/dist/src/archive/copy-loader.js

@@ -0,0 +1,111 @@
+import { createRequire } from "node:module";
+var __defProp = Object.defineProperty;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+
+// src/archive/copy-loader.ts
+import { once } from "node:events";
+function csvField(value) {
+  if (value === null || value === undefined)
+    return "\\N";
+  const text = typeof value === "number" ? String(value) : value;
+  return `"${text.replace(/"/g, '""')}"`;
+}
+function asNumber(value) {
+  return typeof value === "bigint" ? Number(value) : Number(value);
+}
+function asText(value) {
+  return Buffer.isBuffer(value) ? value.toString("utf8") : String(value);
+}
+function asOptionalText(value) {
+  return value === null || value === undefined ? null : asText(value);
+}
+function toCsvLine(dataset, row) {
+  if (dataset === "blocks") {
+    return [
+      csvField(asNumber(row.height)),
+      csvField(asText(row.hash)),
+      csvField(asText(row.parent_hash)),
+      csvField(asNumber(row.burn_block_height)),
+      csvField(asOptionalText(row.burn_block_hash)),
+      csvField(asOptionalText(row.index_block_hash)),
+      csvField(asNumber(row.timestamp))
+    ].join(",");
+  }
+  if (dataset === "transactions") {
+    return [
+      csvField(asText(row.tx_id)),
+      csvField(asNumber(row.block_height)),
+      csvField(asNumber(row.tx_index)),
+      csvField(asText(row.type)),
+      csvField(asText(row.sender)),
+      csvField(asText(row.status)),
+      csvField(asOptionalText(row.contract_id)),
+      csvField(asOptionalText(row.function_name)),
+      csvField(asOptionalText(row.function_args_json)),
+      csvField(asOptionalText(row.raw_result)),
+      csvField(asText(row.raw_tx))
+    ].join(",");
+  }
+  return [
+    csvField(asText(row.tx_id)),
+    csvField(asNumber(row.block_height)),
+    csvField(asNumber(row.event_index)),
+    csvField(asText(row.event_type)),
+    csvField(asText(row.data_json))
+  ].join(",");
+}
+function copyStatement(dataset) {
+  if (dataset === "blocks") {
+    return "COPY blocks (height, hash, parent_hash, burn_block_height, burn_block_hash, index_block_hash, timestamp) FROM STDIN WITH (FORMAT csv, NULL '\\N')";
+  }
+  if (dataset === "transactions") {
+    return "COPY transactions (tx_id, block_height, tx_index, type, sender, status, contract_id, function_name, function_args, raw_result, raw_tx) FROM STDIN WITH (FORMAT csv, NULL '\\N')";
+  }
+  return "COPY events (tx_id, block_height, event_index, type, data) FROM STDIN WITH (FORMAT csv, NULL '\\N')";
+}
+async function writeRowsToCopyStream(params) {
+  const { writable, dataset, rows } = params;
+  const progressEvery = params.progressEvery ?? 500000;
+  let count = 0;
+  try {
+    for await (const row of rows) {
+      const line = `${toCsvLine(dataset, row)}
+`;
+      if (!writable.write(line)) {
+        await once(writable, "drain");
+      }
+      count++;
+      if (params.onProgress && count % progressEvery === 0) {
+        params.onProgress(count);
+      }
+    }
+    await new Promise((resolve, reject) => {
+      writable.end((error) => error ? reject(error) : resolve());
+    });
+  } catch (error) {
+    writable.destroy(error instanceof Error ? error : new Error(String(error)));
+    throw error;
+  }
+  return count;
+}
+export {
+  writeRowsToCopyStream,
+  toCsvLine,
+  csvField,
+  copyStatement
+};
+
+//# debugId=2B81DFB47D25D63C64756E2164756E21
+//# sourceMappingURL=copy-loader.js.map

package/dist/src/archive/copy-loader.js.map

@@ -0,0 +1,10 @@
+{
+  "version": 3,
+  "sources": ["../src/archive/copy-loader.ts"],
+  "sourcesContent": [
+    "import { once } from \"node:events\";\nimport type { Writable } from \"node:stream\";\n\n/**\n * Streaming `COPY ... FROM STDIN` loader for canonical archive rows.\n *\n * Restoring history row-by-row is not viable at chain scale: batched INSERTs\n * measured ~2k rows/s on this schema, which is days for a full genesis restore.\n * COPY moves the same data at ~64k rows/s because it is one statement per\n * partition with no per-row round-trip.\n *\n * This module deliberately knows nothing about Parquet or about where rows come\n * from — it accepts the archive's row shape as an async iterable. That keeps the\n * heavy file-format dependency out of every consumer that only needs to write\n * rows, and lets the CLI and the indexer share one loader rather than growing\n * two subtly different CSV encoders.\n */\n\nexport type ArchiveDataset = \"blocks\" | \"transactions\" | \"events\";\n\n/** Row shapes exactly as the archive's Parquet columns are named. */\nexport type ArchiveRow = Record<string, unknown>;\n\n/**\n * CSV encoding for `WITH (FORMAT csv, NULL '\\N')`: every present value is\n * quoted with embedded quotes doubled, so a value that happens to equal the\n * NULL marker cannot be misread as SQL NULL. Only a genuinely absent value is\n * written unquoted as `\\N`.\n */\nexport function csvField(value: string | number | null | undefined): string {\n\tif (value === null || value === undefined) return \"\\\\N\";\n\tconst text = typeof value === \"number\" ? String(value) : value;\n\treturn `\"${text.replace(/\"/g, '\"\"')}\"`;\n}\n\nfunction asNumber(value: unknown): number {\n\treturn typeof value === \"bigint\" ? Number(value) : Number(value as number);\n}\n\nfunction asText(value: unknown): string {\n\treturn Buffer.isBuffer(value) ? value.toString(\"utf8\") : String(value);\n}\n\nfunction asOptionalText(value: unknown): string | null {\n\treturn value === null || value === undefined ? null : asText(value);\n}\n\nexport function toCsvLine(dataset: ArchiveDataset, row: ArchiveRow): string {\n\tif (dataset === \"blocks\") {\n\t\treturn [\n\t\t\tcsvField(asNumber(row.height)),\n\t\t\tcsvField(asText(row.hash)),\n\t\t\tcsvField(asText(row.parent_hash)),\n\t\t\tcsvField(asNumber(row.burn_block_height)),\n\t\t\tcsvField(asOptionalText(row.burn_block_hash)),\n\t\t\tcsvField(asOptionalText(row.index_block_hash)),\n\t\t\tcsvField(asNumber(row.timestamp)),\n\t\t].join(\",\");\n\t}\n\tif (dataset === \"transactions\") {\n\t\treturn [\n\t\t\tcsvField(asText(row.tx_id)),\n\t\t\tcsvField(asNumber(row.block_height)),\n\t\t\tcsvField(asNumber(row.tx_index)),\n\t\t\tcsvField(asText(row.type)),\n\t\t\tcsvField(asText(row.sender)),\n\t\t\tcsvField(asText(row.status)),\n\t\t\tcsvField(asOptionalText(row.contract_id)),\n\t\t\tcsvField(asOptionalText(row.function_name)),\n\t\t\tcsvField(asOptionalText(row.function_args_json)),\n\t\t\tcsvField(asOptionalText(row.raw_result)),\n\t\t\tcsvField(asText(row.raw_tx)),\n\t\t].join(\",\");\n\t}\n\treturn [\n\t\tcsvField(asText(row.tx_id)),\n\t\tcsvField(asNumber(row.block_height)),\n\t\tcsvField(asNumber(row.event_index)),\n\t\tcsvField(asText(row.event_type)),\n\t\tcsvField(asText(row.data_json)),\n\t].join(\",\");\n}\n\n/**\n * `canonical` is omitted from the blocks column list: every archived block is\n * canonical by definition, and the schema defaults it to true, so carrying it\n * through every row would be dead weight.\n */\nexport function copyStatement(dataset: ArchiveDataset): string {\n\tif (dataset === \"blocks\") {\n\t\treturn \"COPY blocks (height, hash, parent_hash, burn_block_height, burn_block_hash, index_block_hash, timestamp) FROM STDIN WITH (FORMAT csv, NULL '\\\\N')\";\n\t}\n\tif (dataset === \"transactions\") {\n\t\treturn \"COPY transactions (tx_id, block_height, tx_index, type, sender, status, contract_id, function_name, function_args, raw_result, raw_tx) FROM STDIN WITH (FORMAT csv, NULL '\\\\N')\";\n\t}\n\treturn \"COPY events (tx_id, block_height, event_index, type, data) FROM STDIN WITH (FORMAT csv, NULL '\\\\N')\";\n}\n\n/**\n * Stream rows onto an open COPY writable. The caller owns opening it (drivers\n * differ in how a COPY stream is obtained) and this owns the wire format,\n * backpressure, and failure semantics.\n *\n * On error the stream is destroyed rather than ended, so Postgres aborts the\n * COPY instead of committing a partial one — an interrupted load must leave\n * nothing behind, not half a partition.\n */\nexport async function writeRowsToCopyStream(params: {\n\twritable: Writable;\n\tdataset: ArchiveDataset;\n\trows: AsyncIterable<ArchiveRow>;\n\tonProgress?: (rowsWritten: number) => void;\n\tprogressEvery?: number;\n}): Promise<number> {\n\tconst { writable, dataset, rows } = params;\n\tconst progressEvery = params.progressEvery ?? 500_000;\n\tlet count = 0;\n\ttry {\n\t\tfor await (const row of rows) {\n\t\t\tconst line = `${toCsvLine(dataset, row)}\\n`;\n\t\t\tif (!writable.write(line)) {\n\t\t\t\tawait once(writable, \"drain\");\n\t\t\t}\n\t\t\tcount++;\n\t\t\tif (params.onProgress && count % progressEvery === 0) {\n\t\t\t\tparams.onProgress(count);\n\t\t\t}\n\t\t}\n\t\tawait new Promise<void>((resolve, reject) => {\n\t\t\twritable.end((error?: Error | null) =>\n\t\t\t\terror ? reject(error) : resolve(),\n\t\t\t);\n\t\t});\n\t} catch (error) {\n\t\twritable.destroy(error instanceof Error ? error : new Error(String(error)));\n\t\tthrow error;\n\t}\n\treturn count;\n}\n"
+  ],
+  "mappings": ";;;;;;;;;;;;;;;;;AAAA;AA6BO,SAAS,QAAQ,CAAC,OAAmD;AAAA,EAC3E,IAAI,UAAU,QAAQ,UAAU;AAAA,IAAW,OAAO;AAAA,EAClD,MAAM,OAAO,OAAO,UAAU,WAAW,OAAO,KAAK,IAAI;AAAA,EACzD,OAAO,IAAI,KAAK,QAAQ,MAAM,IAAI;AAAA;AAGnC,SAAS,QAAQ,CAAC,OAAwB;AAAA,EACzC,OAAO,OAAO,UAAU,WAAW,OAAO,KAAK,IAAI,OAAO,KAAe;AAAA;AAG1E,SAAS,MAAM,CAAC,OAAwB;AAAA,EACvC,OAAO,OAAO,SAAS,KAAK,IAAI,MAAM,SAAS,MAAM,IAAI,OAAO,KAAK;AAAA;AAGtE,SAAS,cAAc,CAAC,OAA+B;AAAA,EACtD,OAAO,UAAU,QAAQ,UAAU,YAAY,OAAO,OAAO,KAAK;AAAA;AAG5D,SAAS,SAAS,CAAC,SAAyB,KAAyB;AAAA,EAC3E,IAAI,YAAY,UAAU;AAAA,IACzB,OAAO;AAAA,MACN,SAAS,SAAS,IAAI,MAAM,CAAC;AAAA,MAC7B,SAAS,OAAO,IAAI,IAAI,CAAC;AAAA,MACzB,SAAS,OAAO,IAAI,WAAW,CAAC;AAAA,MAChC,SAAS,SAAS,IAAI,iBAAiB,CAAC;AAAA,MACxC,SAAS,eAAe,IAAI,eAAe,CAAC;AAAA,MAC5C,SAAS,eAAe,IAAI,gBAAgB,CAAC;AAAA,MAC7C,SAAS,SAAS,IAAI,SAAS,CAAC;AAAA,IACjC,EAAE,KAAK,GAAG;AAAA,EACX;AAAA,EACA,IAAI,YAAY,gBAAgB;AAAA,IAC/B,OAAO;AAAA,MACN,SAAS,OAAO,IAAI,KAAK,CAAC;AAAA,MAC1B,SAAS,SAAS,IAAI,YAAY,CAAC;AAAA,MACnC,SAAS,SAAS,IAAI,QAAQ,CAAC;AAAA,MAC/B,SAAS,OAAO,IAAI,IAAI,CAAC;AAAA,MACzB,SAAS,OAAO,IAAI,MAAM,CAAC;AAAA,MAC3B,SAAS,OAAO,IAAI,MAAM,CAAC;AAAA,MAC3B,SAAS,eAAe,IAAI,WAAW,CAAC;AAAA,MACxC,SAAS,eAAe,IAAI,aAAa,CAAC;AAAA,MAC1C,SAAS,eAAe,IAAI,kBAAkB,CAAC;AAAA,MAC/C,SAAS,eAAe,IAAI,UAAU,CAAC;AAAA,MACvC,SAAS,OAAO,IAAI,MAAM,CAAC;AAAA,IAC5B,EAAE,KAAK,GAAG;AAAA,EACX;AAAA,EACA,OAAO;AAAA,IACN,SAAS,OAAO,IAAI,KAAK,CAAC;AAAA,IAC1B,SAAS,SAAS,IAAI,YAAY,CAAC;AAAA,IACnC,SAAS,SAAS,IAAI,WAAW,CAAC;AAAA,IAClC,SAAS,OAAO,IAAI,UAAU,CAAC;AAAA,IAC/B,SAAS,OAAO,IAAI,SAAS,CAAC;AAAA,EAC/B,EAAE,KAAK,GAAG;AAAA;AAQJ,SAAS,aAAa,CAAC,SAAiC;AAAA,EAC9D,IAAI,YAAY,UAAU;AAAA,IACzB,OAAO;AAAA,EACR;AAAA,EACA,IAAI,YAAY,gBAAgB;AAAA,IAC/B,OAAO;AAAA,EACR;AAAA,EACA,OAAO;AAAA;AAYR,eAAsB,qBAAqB,CAAC,QAMxB;AAAA,EACnB,QAAQ,UAAU,SAAS,SAAS;AAAA,EACpC,MAAM,gBAAgB,OAAO,iBAAiB;AAAA,EAC9C,IAAI,QAAQ;AAAA,EACZ,IAAI;AAAA,IACH,iBAAiB,OAAO,MAAM;AAAA,MAC7B,MAAM,OAAO,GAAG,UAAU,SAAS,GAAG;AAAA;AAAA,MACtC,IAAI,CAAC,SAAS,MAAM,IAAI,GAAG;AAAA,QAC1B,MAAM,KAAK,UAAU,OAAO;AAAA,MAC7B;AAAA,MACA;AAAA,MACA,IAAI,OAAO,cAAc,QAAQ,kBAAkB,GAAG;AAAA,QACrD,OAAO,WAAW,KAAK;AAAA,MACxB;AAAA,IACD;AAAA,IACA,MAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAAA,MAC5C,SAAS,IAAI,CAAC,UACb,QAAQ,OAAO,KAAK,IAAI,QAAQ,CACjC;AAAA,KACA;AAAA,IACA,OAAO,OAAO;AAAA,IACf,SAAS,QAAQ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC,CAAC;AAAA,IAC1E,MAAM;AAAA;AAAA,EAEP,OAAO;AAAA;",
+  "debugId": "2B81DFB47D25D63C64756E2164756E21",
+  "names": []
+}