npm - @rocicorp/zero - Versions diffs - 0.26.0 → 0.26.1-canary.2 - Mend

@rocicorp/zero 0.26.0 → 0.26.1-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

package/out/zero/package.json.js CHANGED Viewed

@@ -1,4 +1,4 @@
-const version = "0.26.0";
+const version = "0.26.1-canary.2";
 const packageJson = {
   version
 };

package/out/zero-cache/src/observability/events.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/observability/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAGjD~~,OAAO,EAAc,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAKzE,OAAO,EAAC,KAAK,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;~~AAyBjE~~;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,UAAU,EACd,EAAC,MAAM,EAAE,UAAU,EAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,QAAQ,GAAG,YAAY,CAAC,~~QAqD1E~~;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,GAAG,OAAa,QAO1E;AAED,wBAAgB,YAAY,CAAC,CAAC,SAAS,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,QAEzE;AAED,wBAAsB,oBAAoB,CAAC,CAAC,SAAS,SAAS,EAC5D,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,CAAC,iBAGT;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,GAAG,UAAU,CAevD"}
1	+ {"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/observability/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAc,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAKzE,OAAO,EAAC,KAAK,SAAS,EAAC,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AA+BjE;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,UAAU,EACd,EAAC,MAAM,EAAE,UAAU,EAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,QAAQ,GAAG,YAAY,CAAC,QAsE1E;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,GAAG,OAAa,QAO1E;AAED,wBAAgB,YAAY,CAAC,CAAC,SAAS,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,QAEzE;AAED,wBAAsB,oBAAoB,CAAC,CAAC,SAAS,SAAS,EAC5D,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,CAAC,iBAGT;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,OAAO,GAAG,UAAU,CAevD"}

package/out/zero-cache/src/observability/events.js CHANGED Viewed

@@ -1,5 +1,8 @@
+import { resolver } from "@rocicorp/resolver";
 import { emitterFor, httpTransport, CloudEvent } from "cloudevents";
 import { nanoid } from "nanoid";
+import { gzip } from "node:zlib";
+import { stringify } from "../../../shared/src/bigint-json.js";
 import { isJSONValue } from "../../../shared/src/json.js";
 import { must } from "../../../shared/src/must.js";
 import { promiseVoid } from "../../../shared/src/resolved-promises.js";
@@ -17,6 +20,11 @@ let publishFn = (lc, { type }) => {
 const attributeValueSchema = union(string(), number(), boolean());
 const eventSchema = record(attributeValueSchema);
 const extensionsObjectSchema = object({ extensions: eventSchema });
+async function base64gzip(str) {
+  const { promise: gzipped, resolve, reject } = resolver();
+  gzip(Buffer.from(str), (err, buf) => err ? reject(err) : resolve(buf));
+  return (await gzipped).toString("base64");
+}
 function initEventSink(lc, { taskID, cloudEvent }) {
   if (!cloudEvent.sinkEnv) {
     publishFn = (lc2, event) => {
@@ -31,13 +39,21 @@ function initEventSink(lc, { taskID, cloudEvent }) {
     const { extensions } = parse(JSON.parse(strVal), extensionsObjectSchema);
     overrides = extensions;
   }
-  function createCloudEvent(data) {
-    const { type, time } = data;
+  async function createCloudEvent(event) {
+    const { type, time } = event;
+    const json = stringify(event);
+    const data = await base64gzip(json);
     return new CloudEvent({
       id: nanoid(),
       source: taskID,
       type,
       time,
+      // Pass `data` as text/plain to prevent intermediaries from
+      // base64-decoding it. It is the responsibility of the final processor
+      // to recognize that datacontentencoding === "gzip" and unpack the
+      // `data` accordingly before parsing it.
+      datacontenttype: "text/plain",
+      datacontentencoding: "gzip",
       data,
       ...overrides
     });
@@ -46,7 +62,13 @@ function initEventSink(lc, { taskID, cloudEvent }) {
   const emit = emitterFor(httpTransport(sinkURI));
   lc.debug?.(`Publishing ZeroEvents to ${sinkURI}`);
   publishFn = async (lc2, event) => {
-    const cloudEvent2 = createCloudEvent(event);
+    let cloudEvent2;
+    try {
+      cloudEvent2 = await createCloudEvent(event);
+    } catch (e) {
+      lc2.error?.(`Error creating CloudEvent ${event.type}`, e);
+      return;
+    }
     lc2.debug?.(`Publishing CloudEvent: ${cloudEvent2.type}`);
     for (let i = 0; i < MAX_PUBLISH_ATTEMPTS; i++) {
       if (i > 0) {
@@ -54,7 +76,8 @@ function initEventSink(lc, { taskID, cloudEvent }) {
       }
       try {
         await emit(cloudEvent2);
-        lc2.info?.(`Published CloudEvent: ${cloudEvent2.type}`, cloudEvent2);
+        const { data: _, ...event2 } = cloudEvent2;
+        lc2.info?.(`Published CloudEvent: ${cloudEvent2.type}`, { event: event2 });
         return;
       } catch (e) {
         lc2.warn?.(`Error publishing ${cloudEvent2.type} (attempt ${i + 1})`, e);
@@ -62,9 +85,6 @@ function initEventSink(lc, { taskID, cloudEvent }) {
     }
   };
 }
-function publishEvent(lc, event) {
-  void publishFn(lc, event);
-}
 async function publishCriticalEvent(lc, event) {
   await publishFn(lc, event);
 }
@@ -86,7 +106,6 @@ function makeErrorDetails(e) {
 export {
   initEventSink,
   makeErrorDetails,
-  publishCriticalEvent,
-  publishEvent
+  publishCriticalEvent
 };
 //# sourceMappingURL=events.js.map

package/out/zero-cache/src/observability/events.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"events.js","sources":["../../../../../zero-cache/src/observability/events.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {CloudEvent, emitterFor, httpTransport} from 'cloudevents';\nimport {nanoid} from 'nanoid';\nimport {isJSONValue, type JSONObject} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {type ZeroEvent} from '../../../zero-events/src/index.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\n\nconst MAX_PUBLISH_ATTEMPTS = 6;\nconst INITIAL_PUBLISH_BACKOFF_MS = 500;\n\ntype PublisherFn = (lc: LogContext, event: ZeroEvent) => Promise<void>;\n\nlet publishFn: PublisherFn = (lc, {type}) => {\n lc.warn?.(\n `Cannot publish \"${type}\" event before initEventSink(). ` +\n `This is only expected in unit tests.`,\n );\n return promiseVoid;\n};\n\nconst attributeValueSchema = v.union(v.string(), v.number(), v.boolean());\n\nconst eventSchema = v.record(attributeValueSchema);\n\ntype PartialEvent = v.Infer<typeof eventSchema>;\n\n// Note: This conforms to the format of the knative K_CE_OVERRIDES binding:\n// https://github.com/knative/eventing/blob/main/docs/spec/sources.md#sinkbinding\nconst extensionsObjectSchema = v.object({extensions: eventSchema});\n\n/*\n Initializes a per-process event sink according to the cloud event\n * parameters in the ZeroConfig. This must be called at the beginning\n * of the process, before any ZeroEvents are generated / published.\n /\nexport function initEventSink(\n lc: LogContext,\n {taskID, cloudEvent}: Pick<NormalizedZeroConfig, 'taskID' \| 'cloudEvent'>,\n) {\n if (!cloudEvent.sinkEnv) {\n // The default implementation just outputs the events to logs.\n publishFn = (lc, event) => {\n lc.info?.(`ZeroEvent: ${event.type}`, event);\n return promiseVoid;\n };\n return;\n }\n\n let overrides: PartialEvent = {};\n\n if (cloudEvent.extensionOverridesEnv) {\n const strVal = must(process.env[cloudEvent.extensionOverridesEnv]);\n const {extensions} = v.parse(JSON.parse(strVal), extensionsObjectSchema);\n overrides = extensions;\n }\n\n function createCloudEvent(~~data~~: ZeroEvent) {\n const {type, time} = data;\n return new CloudEvent({\n id: nanoid(),\n source: taskID,\n type,\n time,\n data,\n ...overrides,\n });\n }\n\n const sinkURI = must(process.env[cloudEvent.sinkEnv]);\n const emit = emitterFor(httpTransport(sinkURI));\n lc.debug?.(`Publishing ZeroEvents to ${sinkURI}`);\n\n publishFn = async (lc, event) => {\n ~~const~~ cloudEvent = createCloudEvent(event);\n lc.debug?.(`Publishing CloudEvent: ${cloudEvent.type}`);\n\n for (let i = 0; i < MAX_PUBLISH_ATTEMPTS; i++) {\n if (i > 0) {\n // exponential backoff on retries\n await sleep(INITIAL_PUBLISH_BACKOFF_MS 2 ** (i - 1));\n }\n try {\n await emit(cloudEvent);\n lc.info?.(`Published CloudEvent: ${cloudEvent.type}`, ~~cloudEvent~~);\n return;\n } catch (e) {\n lc.warn?.(`Error publishing ${cloudEvent.type} (attempt ${i + 1})`, e);\n }\n }\n };\n}\n\nexport function initEventSinkForTesting(sink: ZeroEvent[], now = new Date()) {\n publishFn = (lc, event) => {\n lc.info?.(`Testing event sink received ${event.type} event`, event);\n // Replace the default Date.now() with the test instance for determinism.\n sink.push({...event, time: now.toISOString()});\n return promiseVoid;\n };\n}\n\nexport function publishEvent<E extends ZeroEvent>(lc: LogContext, event: E) {\n void publishFn(lc, event);\n}\n\nexport async function publishCriticalEvent<E extends ZeroEvent>(\n lc: LogContext,\n event: E,\n) {\n await publishFn(lc, event);\n}\n\nexport function makeErrorDetails(e: unknown): JSONObject {\n const err = e instanceof Error ? e : new Error(String(e));\n const errorDetails: JSONObject = {\n name: err.name,\n message: err.message,\n stack: err.stack,\n cause: err.cause ? makeErrorDetails(err.cause) : undefined,\n };\n // Include any enumerable properties (e.g. of Error subtypes).\n for (const [field, value] of Object.entries(err)) {\n if (isJSONValue(value, [])) {\n errorDetails[field] = value;\n }\n }\n return errorDetails;\n}\n"],"names":["v.union","v.string","v.number","v.boolean","v.record","v.object","lc","v.parse","cloudEvent"],"mappings":"~~;;;;;;;;AAWA~~,MAAM,uBAAuB;AAC7B,MAAM,6BAA6B;AAInC,IAAI,YAAyB,CAAC,IAAI,EAAC,WAAU;AAC3C,KAAG;AAAA,IACD,mBAAmB,IAAI;AAAA,EAAA;AAGzB,SAAO;AACT;AAEA,MAAM,uBAAuBA,MAAQC,OAAE,GAAUC,OAAE,GAAUC,QAAE,CAAS;AAExE,MAAM,cAAcC,OAAS,oBAAoB;AAMjD,MAAM,yBAAyBC,OAAS,EAAC,YAAY,aAAY;~~AAO1D~~,SAAS,cACd,IACA,EAAC,QAAQ,cACT;AACA,MAAI,CAAC,WAAW,SAAS;AAEvB,gBAAY,CAACC,KAAI,UAAU;AACzBA,UAAG,OAAO,cAAc,MAAM,IAAI,IAAI,KAAK;AAC3C,aAAO;AAAA,IACT;AACA;AAAA,EACF;AAEA,MAAI,YAA0B,CAAA;AAE9B,MAAI,WAAW,uBAAuB;AACpC,UAAM,SAAS,KAAK,QAAQ,IAAI,WAAW,qBAAqB,CAAC;AACjE,UAAM,EAAC,eAAcC,MAAQ,KAAK,MAAM,MAAM,GAAG,sBAAsB;AACvE,gBAAY;AAAA,EACd;AAEA,~~WAAS~~,iBAAiB,~~MAAiB~~;~~AACzC~~,UAAM,EAAC,MAAM,KAAA,IAAQ;AACrB,WAAO,IAAI,WAAW;AAAA,MACpB,IAAI,OAAA;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,~~MACA~~;AAAA,MACA,GAAG;AAAA,IAAA,CACJ;AAAA,EACH;AAEA,QAAM,UAAU,KAAK,QAAQ,IAAI,WAAW,OAAO,CAAC;AACpD,QAAM,OAAO,WAAW,cAAc,OAAO,CAAC;AAC9C,KAAG,QAAQ,4BAA4B,OAAO,EAAE;AAEhD,cAAY,OAAOD,KAAI,UAAU;AAC/B,~~UAAME~~,~~cAAa~~,iBAAiB,KAAK;~~AACzCF~~,QAAG,QAAQ,0BAA0BE,YAAW,IAAI,EAAE;AAEtD,aAAS,IAAI,GAAG,IAAI,sBAAsB,KAAK;AAC7C,UAAI,IAAI,GAAG;AAET,cAAM,MAAM,6BAA6B,MAAM,IAAI,EAAE;AAAA,MACvD;AACA,UAAI;AACF,cAAM,KAAKA,WAAU;~~AACrBF~~,YAAG,OAAO,yBAAyBE,YAAW,IAAI,~~IAAIA~~,~~WAAU~~;~~AAChE~~;AAAA,MACF,SAAS,GAAG;~~AACVF~~,YAAG,OAAO,oBAAoBE,YAAW,IAAI,aAAa,IAAI,CAAC,KAAK,CAAC;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AACF;~~AAWO~~,~~SAAS,aAAkC,IAAgB,OAAU;AAC1E,OAAK,UAAU,IAAI,KAAK;AAC1B;AAEA,~~eAAsB,qBACpB,IACA,OACA;AACA,QAAM,UAAU,IAAI,KAAK;AAC3B;AAEO,SAAS,iBAAiB,GAAwB;AACvD,QAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC;AACxD,QAAM,eAA2B;AAAA,IAC/B,MAAM,IAAI;AAAA,IACV,SAAS,IAAI;AAAA,IACb,OAAO,IAAI;AAAA,IACX,OAAO,IAAI,QAAQ,iBAAiB,IAAI,KAAK,IAAI;AAAA,EAAA;AAGnD,aAAW,CAAC,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAChD,QAAI,YAAY,OAAO,CAAA,CAAE,GAAG;AAC1B,mBAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AACA,SAAO;AACT;"}
1	+ {"version":3,"file":"events.js","sources":["../../../../../zero-cache/src/observability/events.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {CloudEvent, emitterFor, httpTransport} from 'cloudevents';\nimport {nanoid} from 'nanoid';\nimport {gzip} from 'node:zlib';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {isJSONValue, type JSONObject} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {type ZeroEvent} from '../../../zero-events/src/index.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\n\nconst MAX_PUBLISH_ATTEMPTS = 6;\nconst INITIAL_PUBLISH_BACKOFF_MS = 500;\n\ntype PublisherFn = (lc: LogContext, event: ZeroEvent) => Promise<void>;\n\nlet publishFn: PublisherFn = (lc, {type}) => {\n lc.warn?.(\n `Cannot publish \"${type}\" event before initEventSink(). ` +\n `This is only expected in unit tests.`,\n );\n return promiseVoid;\n};\n\nconst attributeValueSchema = v.union(v.string(), v.number(), v.boolean());\n\nconst eventSchema = v.record(attributeValueSchema);\n\ntype PartialEvent = v.Infer<typeof eventSchema>;\n\n// Note: This conforms to the format of the knative K_CE_OVERRIDES binding:\n// https://github.com/knative/eventing/blob/main/docs/spec/sources.md#sinkbinding\nconst extensionsObjectSchema = v.object({extensions: eventSchema});\n\nasync function base64gzip(str: string): Promise<string> {\n const {promise: gzipped, resolve, reject} = resolver<Buffer>();\n gzip(Buffer.from(str), (err, buf) => (err ? reject(err) : resolve(buf)));\n return (await gzipped).toString('base64');\n}\n\n/*\n Initializes a per-process event sink according to the cloud event\n * parameters in the ZeroConfig. This must be called at the beginning\n * of the process, before any ZeroEvents are generated / published.\n /\nexport function initEventSink(\n lc: LogContext,\n {taskID, cloudEvent}: Pick<NormalizedZeroConfig, 'taskID' \| 'cloudEvent'>,\n) {\n if (!cloudEvent.sinkEnv) {\n // The default implementation just outputs the events to logs.\n publishFn = (lc, event) => {\n lc.info?.(`ZeroEvent: ${event.type}`, event);\n return promiseVoid;\n };\n return;\n }\n\n let overrides: PartialEvent = {};\n\n if (cloudEvent.extensionOverridesEnv) {\n const strVal = must(process.env[cloudEvent.extensionOverridesEnv]);\n const {extensions} = v.parse(JSON.parse(strVal), extensionsObjectSchema);\n overrides = extensions;\n }\n\n async function createCloudEvent(event: ZeroEvent) {\n const {type, time} = event;\n const json = stringify(event);\n const data = await base64gzip(json);\n\n return new CloudEvent({\n id: nanoid(),\n source: taskID,\n type,\n time,\n // Pass `data` as text/plain to prevent intermediaries from\n // base64-decoding it. It is the responsibility of the final processor\n // to recognize that datacontentencoding === \"gzip\" and unpack the\n // `data` accordingly before parsing it.\n datacontenttype: 'text/plain',\n datacontentencoding: 'gzip',\n data,\n ...overrides,\n });\n }\n\n const sinkURI = must(process.env[cloudEvent.sinkEnv]);\n const emit = emitterFor(httpTransport(sinkURI));\n lc.debug?.(`Publishing ZeroEvents to ${sinkURI}`);\n\n publishFn = async (lc, event) => {\n let cloudEvent: CloudEvent<string>;\n try {\n cloudEvent = await createCloudEvent(event);\n } catch (e) {\n lc.error?.(`Error creating CloudEvent ${event.type}`, e);\n return;\n }\n lc.debug?.(`Publishing CloudEvent: ${cloudEvent.type}`);\n\n for (let i = 0; i < MAX_PUBLISH_ATTEMPTS; i++) {\n if (i > 0) {\n // exponential backoff on retries\n await sleep(INITIAL_PUBLISH_BACKOFF_MS 2 ** (i - 1));\n }\n try {\n await emit(cloudEvent);\n // Avoid logging the (possibly large and) unreadable data field.\n const {data: _, ...event} = cloudEvent;\n lc.info?.(`Published CloudEvent: ${cloudEvent.type}`, {event});\n return;\n } catch (e) {\n lc.warn?.(`Error publishing ${cloudEvent.type} (attempt ${i + 1})`, e);\n }\n }\n };\n}\n\nexport function initEventSinkForTesting(sink: ZeroEvent[], now = new Date()) {\n publishFn = (lc, event) => {\n lc.info?.(`Testing event sink received ${event.type} event`, event);\n // Replace the default Date.now() with the test instance for determinism.\n sink.push({...event, time: now.toISOString()});\n return promiseVoid;\n };\n}\n\nexport function publishEvent<E extends ZeroEvent>(lc: LogContext, event: E) {\n void publishFn(lc, event);\n}\n\nexport async function publishCriticalEvent<E extends ZeroEvent>(\n lc: LogContext,\n event: E,\n) {\n await publishFn(lc, event);\n}\n\nexport function makeErrorDetails(e: unknown): JSONObject {\n const err = e instanceof Error ? e : new Error(String(e));\n const errorDetails: JSONObject = {\n name: err.name,\n message: err.message,\n stack: err.stack,\n cause: err.cause ? makeErrorDetails(err.cause) : undefined,\n };\n // Include any enumerable properties (e.g. of Error subtypes).\n for (const [field, value] of Object.entries(err)) {\n if (isJSONValue(value, [])) {\n errorDetails[field] = value;\n }\n }\n return errorDetails;\n}\n"],"names":["v.union","v.string","v.number","v.boolean","v.record","v.object","lc","v.parse","cloudEvent","event"],"mappings":";;;;;;;;;;;AAcA,MAAM,uBAAuB;AAC7B,MAAM,6BAA6B;AAInC,IAAI,YAAyB,CAAC,IAAI,EAAC,WAAU;AAC3C,KAAG;AAAA,IACD,mBAAmB,IAAI;AAAA,EAAA;AAGzB,SAAO;AACT;AAEA,MAAM,uBAAuBA,MAAQC,OAAE,GAAUC,OAAE,GAAUC,QAAE,CAAS;AAExE,MAAM,cAAcC,OAAS,oBAAoB;AAMjD,MAAM,yBAAyBC,OAAS,EAAC,YAAY,aAAY;AAEjE,eAAe,WAAW,KAA8B;AACtD,QAAM,EAAC,SAAS,SAAS,SAAS,OAAA,IAAU,SAAA;AAC5C,OAAK,OAAO,KAAK,GAAG,GAAG,CAAC,KAAK,QAAS,MAAM,OAAO,GAAG,IAAI,QAAQ,GAAG,CAAE;AACvE,UAAQ,MAAM,SAAS,SAAS,QAAQ;AAC1C;AAOO,SAAS,cACd,IACA,EAAC,QAAQ,cACT;AACA,MAAI,CAAC,WAAW,SAAS;AAEvB,gBAAY,CAACC,KAAI,UAAU;AACzBA,UAAG,OAAO,cAAc,MAAM,IAAI,IAAI,KAAK;AAC3C,aAAO;AAAA,IACT;AACA;AAAA,EACF;AAEA,MAAI,YAA0B,CAAA;AAE9B,MAAI,WAAW,uBAAuB;AACpC,UAAM,SAAS,KAAK,QAAQ,IAAI,WAAW,qBAAqB,CAAC;AACjE,UAAM,EAAC,eAAcC,MAAQ,KAAK,MAAM,MAAM,GAAG,sBAAsB;AACvE,gBAAY;AAAA,EACd;AAEA,iBAAe,iBAAiB,OAAkB;AAChD,UAAM,EAAC,MAAM,KAAA,IAAQ;AACrB,UAAM,OAAO,UAAU,KAAK;AAC5B,UAAM,OAAO,MAAM,WAAW,IAAI;AAElC,WAAO,IAAI,WAAW;AAAA,MACpB,IAAI,OAAA;AAAA,MACJ,QAAQ;AAAA,MACR;AAAA,MACA;AAAA;AAAA;AAAA;AAAA;AAAA,MAKA,iBAAiB;AAAA,MACjB,qBAAqB;AAAA,MACrB;AAAA,MACA,GAAG;AAAA,IAAA,CACJ;AAAA,EACH;AAEA,QAAM,UAAU,KAAK,QAAQ,IAAI,WAAW,OAAO,CAAC;AACpD,QAAM,OAAO,WAAW,cAAc,OAAO,CAAC;AAC9C,KAAG,QAAQ,4BAA4B,OAAO,EAAE;AAEhD,cAAY,OAAOD,KAAI,UAAU;AAC/B,QAAIE;AACJ,QAAI;AACFA,oBAAa,MAAM,iBAAiB,KAAK;AAAA,IAC3C,SAAS,GAAG;AACVF,UAAG,QAAQ,6BAA6B,MAAM,IAAI,IAAI,CAAC;AACvD;AAAA,IACF;AACAA,QAAG,QAAQ,0BAA0BE,YAAW,IAAI,EAAE;AAEtD,aAAS,IAAI,GAAG,IAAI,sBAAsB,KAAK;AAC7C,UAAI,IAAI,GAAG;AAET,cAAM,MAAM,6BAA6B,MAAM,IAAI,EAAE;AAAA,MACvD;AACA,UAAI;AACF,cAAM,KAAKA,WAAU;AAErB,cAAM,EAAC,MAAM,GAAG,GAAGC,WAASD;AAC5BF,YAAG,OAAO,yBAAyBE,YAAW,IAAI,IAAI,EAAC,OAAAC,QAAM;AAC7D;AAAA,MACF,SAAS,GAAG;AACVH,YAAG,OAAO,oBAAoBE,YAAW,IAAI,aAAa,IAAI,CAAC,KAAK,CAAC;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AACF;AAeA,eAAsB,qBACpB,IACA,OACA;AACA,QAAM,UAAU,IAAI,KAAK;AAC3B;AAEO,SAAS,iBAAiB,GAAwB;AACvD,QAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,MAAM,OAAO,CAAC,CAAC;AACxD,QAAM,eAA2B;AAAA,IAC/B,MAAM,IAAI;AAAA,IACV,SAAS,IAAI;AAAA,IACb,OAAO,IAAI;AAAA,IACX,OAAO,IAAI,QAAQ,iBAAiB,IAAI,KAAK,IAAI;AAAA,EAAA;AAGnD,aAAW,CAAC,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAChD,QAAI,YAAY,OAAO,CAAA,CAAE,GAAG;AAC1B,mBAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AACA,SAAO;AACT;"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,~~EAEf~~,eAAe,EAChB,MAAM,wBAAwB,CAAC;~~AAWhC~~,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAwDrD"}
1	+ {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EAGf,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAYhC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAwDrD"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -9,7 +9,7 @@ import { TransactionPool } from "../../../db/transaction-pool.js";
 import { pgClient } from "../../../types/pg.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";
 import { tableMetadataSchema, columnMetadataSchema } from "./backfill-metadata.js";
-import { makeSelectPublishedStmt, createReplicationSlot } from "./initial-sync.js";
+import { makeDownloadStatements, createReplicationSlot } from "./initial-sync.js";
 import { toStateVersionString } from "./lsn.js";
 import { getPublicationInfo } from "./schema/published.js";
 const POSTGRES_COPY_CHUNK_SIZE = 64 * 1024;
@@ -35,7 +35,7 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
       lc,
       tx,
       backfill,
-      makeSelectPublishedStmt(tableSpec, cols),
+      makeDownloadStatements(tableSpec, cols),
       cols.map((col) => types.getTypeParser(tableSpec.columns[col].typeOID)),
       flushThresholdBytes
     );
@@ -51,21 +51,34 @@ async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts
     void db.end().catch((e) => lc.warn?.(`error closing backfill connection`, e));
   }
 }
-async function* stream(lc, tx, backfill, selectStmt, colParsers, flushThresholdBytes) {
+async function* stream(lc, tx, backfill, { select, getTotalRows, getTotalBytes }, colParsers, flushThresholdBytes) {
   const start = performance.now();
-  lc.info?.(`Starting backfill copy stream:`, selectStmt);
+  const [rows, bytes] = await tx.processReadTask(
+    (sql) => Promise.all([
+      sql.unsafe(getTotalRows),
+      sql.unsafe(getTotalBytes)
+    ])
+  );
+  const status = {
+    rows: 0,
+    totalRows: Number(rows[0].totalRows),
+    totalBytes: Number(bytes[0].totalBytes)
+  };
+  let elapsed = (performance.now() - start).toFixed(3);
+  lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {
+    status
+  });
   const copyStream = await tx.processReadTask(
-    (sql) => sql.unsafe(`COPY (${selectStmt}) TO STDOUT`).readable()
+    (sql) => sql.unsafe(`COPY (${select}) TO STDOUT`).readable()
   );
   const tsvParser = new TsvParser();
-  let totalRows = 0;
   let totalBytes = 0;
   let totalMsgs = 0;
   let rowValues = [];
   let bufferedBytes = 0;
   const logFlushed = () => {
     lc.debug?.(
-      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${totalRows}, msgs=${totalMsgs}, bytes=${totalBytes})`
+      `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes (total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`
     );
   };
   let row = Array.from({ length: colParsers.length });
@@ -76,7 +89,7 @@ async function* stream(lc, tx, backfill, selectStmt, colParsers, flushThresholdB
       row[col] = text === null ? null : colParsers[col](text);
       if (++col === colParsers.length) {
         rowValues.push(row);
-        totalRows++;
+        status.rows++;
         row = Array.from({ length: colParsers.length });
         col = 0;
       }
@@ -84,7 +97,7 @@ async function* stream(lc, tx, backfill, selectStmt, colParsers, flushThresholdB
     bufferedBytes += chunk.byteLength;
     totalBytes += chunk.byteLength;
     if (bufferedBytes >= flushThresholdBytes) {
-      yield { tag: "backfill", ...backfill, rowValues };
+      yield { tag: "backfill", ...backfill, rowValues, status };
       totalMsgs++;
       logFlushed();
       rowValues = [];
@@ -92,14 +105,14 @@ async function* stream(lc, tx, backfill, selectStmt, colParsers, flushThresholdB
     }
   }
   if (rowValues.length > 0) {
-    yield { tag: "backfill", ...backfill, rowValues };
+    yield { tag: "backfill", ...backfill, rowValues, status };
     totalMsgs++;
     logFlushed();
   }
-  yield { tag: "backfill-completed", ...backfill };
-  const elapsed = performance.now() - start;
+  yield { tag: "backfill-completed", ...backfill, status };
+  elapsed = (performance.now() - start).toFixed(3);
   lc.info?.(
-    `Finished streaming ${totalRows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed.toFixed(3)} ms)`
+    `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`
   );
 }
 async function setSnapshot(lc, upstreamURI, tx, slotNamePrefix) {

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n createReplicationSlot,\n makeSelectPublishedStmt,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n const db = pgClient(lc, upstreamURI, {\n connection: {['application_name']: 'backfill-stream'},\n });\n const tx = new TransactionPool(lc, READONLY).run(db);\n try {\n const watermark = await setSnapshot(lc, upstreamURI, tx, slot);\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n\n yield* stream(\n lc,\n tx,\n backfill,\n makeSelectPublishedStmt(tableSpec, cols),\n cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n flushThresholdBytes,\n );\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx.setDone();\n // errors are already thrown and handled from processReadTask()\n void tx.done().catch(() => {});\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n selectStmt: string,\n colParsers: TypeParser[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n const start = performance.now();\n lc.info?.(`Starting backfill copy stream:`, selectStmt);\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(`COPY (${selectStmt}) TO STDOUT`).readable(),\n );\n\n const tsvParser = new TsvParser();\n let totalRows = 0;\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${totalRows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: colParsers.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const text of tsvParser.parse(chunk)) {\n row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n if (++col === colParsers.length) {\n rowValues.push(row);\n totalRows++;\n row = Array.from({length: colParsers.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill};\n const elapsed = performance.now() - start;\n lc.info?.(\n `Finished streaming ${totalRows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed.toFixed(3)} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function setSnapshot(\n lc: LogContext,\n upstreamURI: string,\n tx: TransactionPool,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(lc, upstreamURI, {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, tempSlot);\n\n await tx.processReadTask(sql =>\n sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`),\n );\n // Once the snapshot has been set, the replication session and slot can\n // be closed / dropped.\n await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return watermark;\n } catch (e) {\n // In the event of a failure, clean up the replication slot if created.\n await replicationSession.unsafe(\n /sql*/\n `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = '${tempSlot}'`,\n );\n lc.error?.(`Failed to create backfill snapshot`, e);\n throw e;\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"names":["spec","v.parse"],"mappings":";;;;;;;;;;;;;;AA8CA,MAAM,2BAA2B,KAAK;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,IAC+B;AACrD,OAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;AAErC,QAAM,EAAC,sBAAsB,yBAAA,IAA4B;AACzD,QAAM,KAAK,SAAS,IAAI,aAAa;AAAA,IACnC,YAAY,EAAC,CAAC,kBAAkB,GAAG,kBAAA;AAAA,EAAiB,CACrD;AACD,QAAM,KAAK,IAAI,gBAAgB,IAAI,QAAQ,EAAE,IAAI,EAAE;AACnD,MAAI;AACF,UAAM,YAAY,MAAM,YAAY,IAAI,aAAa,IAAI,IAAI;AAC7D,UAAM,EAAC,WAAW,SAAA,IAAY,MAAM;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AAGjE,UAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,UAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;AAEpD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,wBAAwB,WAAW,IAAI;AAAA,MACvC,KAAK,IAAI,CAAA,QAAO,MAAM,cAAc,UAAU,QAAQ,GAAG,EAAE,OAAO,CAAC;AAAA,MACnE;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AASV,QACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAC7C;AACA,YAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,GAAE;AAAA,IAChE;AACA,UAAM;AAAA,EACR,UAAA;AACE,OAAG,QAAA;AAEH,SAAK,GAAG,OAAO,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7B,SAAK,GAAG,MAAM,MAAM,OAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;AAAA,EAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,YACA,YACA,qBACqD;AACrD,QAAM,QAAQ,YAAY,IAAA;AAC1B,KAAG,OAAO,kCAAkC,UAAU;AACtD,QAAM,aAAa,MAAM,GAAG;AAAA,IAAgB,SAC1C,IAAI,OAAO,SAAS,UAAU,aAAa,EAAE,SAAA;AAAA,EAAS;AAGxD,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,YAAY;AAChB,MAAI,aAAa;AACjB,MAAI,YAAY;AAChB,MAAI,YAA2B,CAAA;AAC/B,MAAI,gBAAgB;AAEpB,QAAM,aAAa,MAAM;AACvB,OAAG;AAAA,MACD,WAAW,UAAU,MAAM,UAAU,aAAa,uBAChC,SAAS,UAAU,SAAS,WAAW,UAAU;AAAA,IAAA;AAAA,EAEvE;AAGA,MAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC7D,MAAI,MAAM;AAEV,mBAAiB,QAAQ,YAAY;AACnC,UAAM,QAAQ;AACd,eAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,UAAI,GAAG,IAAI,SAAS,OAAO,OAAQ,WAAW,GAAG,EAAE,IAAI;AAEvD,UAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,kBAAU,KAAK,GAAG;AAClB;AACA,cAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC5C,cAAM;AAAA,MACR;AAAA,IACF;AACA,qBAAiB,MAAM;AACvB,kBAAc,MAAM;AAEpB,QAAI,iBAAiB,qBAAqB;AACxC,YAAM,EAAC,KAAK,YAAY,GAAG,UAAU,UAAA;AACrC;AACA,iBAAA;AACA,kBAAY,CAAA;AACZ,sBAAgB;AAAA,IAClB;AAAA,EACF;AAGA,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,EAAC,KAAK,YAAY,GAAG,UAAU,UAAA;AACrC;AACA,eAAA;AAAA,EACF;AAEA,QAAM,EAAC,KAAK,sBAAsB,GAAG,SAAA;AACrC,QAAM,UAAU,YAAY,IAAA,IAAQ;AACpC,KAAG;AAAA,IACD,sBAAsB,SAAS,UAAU,SAAS,UAAU,UAAU,WAChE,QAAQ,QAAQ,CAAC,CAAC;AAAA,EAAA;AAE5B;AAWA,eAAe,YACb,IACA,aACA,IACA,gBACA;AACA,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,GAAG,cAAc,OAAO,KAAK,KAAK;AACnD,MAAI;AACF,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AAE9D,UAAM,GAAG;AAAA,MAAgB,CAAA,QACvB,IAAI,OAAO,6BAA6B,QAAQ,GAAG;AAAA,IAAA;AAIrD,UAAM,mBAAmB,OAAO,0BAA0B,QAAQ,GAAG;AAErE,UAAM,YAAY,qBAAqB,GAAG;AAC1C,OAAG,OAAO,sCAAsC,GAAG,KAAK,SAAS,GAAG;AACpE,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,UAAM,mBAAmB;AAAA;AAAA,MAEvB;AAAA,8BACwB,QAAQ;AAAA,IAAA;AAElC,OAAG,QAAQ,sCAAsC,CAAC;AAClD,UAAM;AAAA,EACR,UAAA;AACE,UAAM,mBAAmB,IAAA;AAAA,EAC3B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,SAAO,GAAG,gBAAgB,OAAM,QAAO;AACrC,UAAM,EAAC,OAAA,IAAU,MAAM,mBAAmB,KAAK,YAAY;AAC3D,UAAM,OAAO,OAAO;AAAA,MAClB,CAAAA,UAAQA,MAAK,WAAW,GAAG,MAAM,UAAUA,MAAK,SAAS,GAAG,MAAM;AAAA,IAAA;AAEpE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,YAAYC,MAAQ,GAAG,MAAM,UAAU,mBAAmB;AAChE,QAAI,KAAK,cAAc,UAAU,WAAW;AAC1C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QAAI,KAAK,QAAQ,UAAU,aAAa;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QACE,CAAC;AAAA,MACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC;AAAA,MACrC,IAAI,IAAI,KAAK,sBAAsB;AAAA,IAAA,GAErC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,UAAU;AAAA,MACd,GAAG,OAAO,QAAQ,UAAU,MAAM;AAAA,MAClC,GAAG,OAAO,QAAQ,GAAG,OAAO;AAAA,IAAA;AAE9B,eAAW,CAAC,KAAK,GAAG,KAAK,SAAS;AAChC,YAAM,UAAU,KAAK,QAAQ,GAAG;AAChC,UAAI,CAAC,SAAS;AACZ,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AACA,YAAM,UAAUA,MAAQ,KAAK,oBAAoB;AACjD,UAAI,QAAQ,WAAW,QAAQ,KAAK;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AAAA,IACF;AACA,UAAM,WAA2B;AAAA,MAC/B,UAAU;AAAA,QACR,QAAQ,GAAG,MAAM;AAAA,QACjB,MAAM,GAAG,MAAM;AAAA,QACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAA;AAAA,MAAC;AAAA,MAEjD,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE;AAAA,QAC/B,CAAA,QAAO,EAAE,OAAO,UAAU;AAAA,MAAA;AAAA,MAE5B;AAAA,IAAA;AAEF,WAAO,EAAC,WAAW,MAAM,SAAA;AAAA,EAC3B,CAAC;AACH;"}
1	+ {"version":3,"file":"backfill-stream.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"sourcesContent":["import {\n PG_UNDEFINED_COLUMN,\n PG_UNDEFINED_TABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {READONLY} from '../../../db/mode-enum.ts';\nimport {TsvParser} from '../../../db/pg-copy.ts';\nimport {getTypeParsers, type TypeParser} from '../../../db/pg-type-parser.ts';\nimport type {PublishedTableSpec} from '../../../db/specs.ts';\nimport {TransactionPool} from '../../../db/transaction-pool.ts';\nimport {pgClient} from '../../../types/pg.ts';\nimport {SchemaIncompatibilityError} from '../common/backfill-manager.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n DownloadStatus,\n JSONValue,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport {\n columnMetadataSchema,\n tableMetadataSchema,\n} from './backfill-metadata.ts';\nimport {\n createReplicationSlot,\n makeDownloadStatements,\n type DownloadStatements,\n} from './initial-sync.ts';\nimport {toStateVersionString} from './lsn.ts';\nimport {getPublicationInfo} from './schema/published.ts';\nimport type {Replica} from './schema/shard.ts';\n\ntype BackfillParams = Omit<BackfillCompleted, 'tag'>;\n\ntype StreamOptions = {\n /*\n The number of bytes at which to flush a batch of rows in a\n * backfill message. Defaults to Node's getDefaultHighWatermark().\n /\n flushThresholdBytes?: number;\n};\n\n// The size of chunks that Postgres sends on COPY stream.\n// This happens to match NodeJS's getDefaultHighWatermark()\n// (for Node v20+).\nconst POSTGRES_COPY_CHUNK_SIZE = 64 1024;\n\n/*\n Streams a series of `backfill` messages (ending with `backfill-complete`)\n * at a set watermark (i.e. LSN). The data is retrieved via a COPY stream\n * made at a transaction snapshot corresponding to specific LSN, obtained by\n * creating a short-lived replication slot.\n /\nexport async function streamBackfill(\n lc: LogContext,\n upstreamURI: string,\n {slot, publications}: Pick<Replica, 'slot' \| 'publications'>,\n bf: BackfillRequest,\n opts: StreamOptions = {},\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n lc = lc\n .withContext('component', 'backfill')\n .withContext('table', bf.table.name);\n\n const {flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE} = opts;\n const db = pgClient(lc, upstreamURI, {\n connection: {['application_name']: 'backfill-stream'},\n });\n const tx = new TransactionPool(lc, READONLY).run(db);\n try {\n const watermark = await setSnapshot(lc, upstreamURI, tx, slot);\n const {tableSpec, backfill} = await validateSchema(\n tx,\n publications,\n bf,\n watermark,\n );\n const types = await getTypeParsers(db, {returnJsonAsString: true});\n\n // Note: validateSchema ensures that the rowKey and columns are disjoint\n const {relation, columns} = backfill;\n const cols = [...relation.rowKey.columns, ...columns];\n\n yield* stream(\n lc,\n tx,\n backfill,\n makeDownloadStatements(tableSpec, cols),\n cols.map(col => types.getTypeParser(tableSpec.columns[col].typeOID)),\n flushThresholdBytes,\n );\n } catch (e) {\n // Although we make the best effort to validate the schema at the\n // transaction snapshot, certain forms of `ALTER TABLE` are not\n // MVCC safe and not \"frozen\" in the snapshot:\n //\n // https://www.postgresql.org/docs/current/mvcc-caveats.html\n //\n // Handle these errors as schema incompatibility errors rather than\n // unknown runtime errors.\n if (\n e instanceof postgres.PostgresError &&\n (e.code === PG_UNDEFINED_TABLE \|\| e.code === PG_UNDEFINED_COLUMN)\n ) {\n throw new SchemaIncompatibilityError(bf, String(e), {cause: e});\n }\n throw e;\n } finally {\n tx.setDone();\n // errors are already thrown and handled from processReadTask()\n void tx.done().catch(() => {});\n // Workaround postgres.js hanging at the end of some COPY commands:\n // https://github.com/porsager/postgres/issues/499\n void db.end().catch(e => lc.warn?.(`error closing backfill connection`, e));\n }\n}\n\nasync function* stream(\n lc: LogContext,\n tx: TransactionPool,\n backfill: BackfillParams,\n {select, getTotalRows, getTotalBytes}: DownloadStatements,\n colParsers: TypeParser[],\n flushThresholdBytes: number,\n): AsyncGenerator<MessageBackfill \| BackfillCompleted> {\n const start = performance.now();\n const [rows, bytes] = await tx.processReadTask(sql =>\n Promise.all([\n sql.unsafe<{totalRows: bigint}[]>(getTotalRows),\n sql.unsafe<{totalBytes: bigint}[]>(getTotalBytes),\n ]),\n );\n const status: DownloadStatus = {\n rows: 0,\n totalRows: Number(rows[0].totalRows),\n totalBytes: Number(bytes[0].totalBytes),\n };\n\n let elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(`Computed total rows and bytes for: ${select} (${elapsed} ms)`, {\n status,\n });\n const copyStream = await tx.processReadTask(sql =>\n sql.unsafe(`COPY (${select}) TO STDOUT`).readable(),\n );\n\n const tsvParser = new TsvParser();\n let totalBytes = 0;\n let totalMsgs = 0;\n let rowValues: JSONValue[][] = [];\n let bufferedBytes = 0;\n\n const logFlushed = () => {\n lc.debug?.(\n `Flushed ${rowValues.length} rows, ${bufferedBytes} bytes ` +\n `(total: rows=${status.rows}, msgs=${totalMsgs}, bytes=${totalBytes})`,\n );\n };\n\n // Tracks the row being parsed.\n let row: JSONValue[] = Array.from({length: colParsers.length});\n let col = 0;\n\n for await (const data of copyStream) {\n const chunk = data as Buffer;\n for (const text of tsvParser.parse(chunk)) {\n row[col] = text === null ? null : (colParsers[col](text) as JSONValue);\n\n if (++col === colParsers.length) {\n rowValues.push(row);\n status.rows++;\n row = Array.from({length: colParsers.length});\n col = 0;\n }\n }\n bufferedBytes += chunk.byteLength;\n totalBytes += chunk.byteLength;\n\n if (bufferedBytes >= flushThresholdBytes) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n rowValues = [];\n bufferedBytes = 0;\n }\n }\n\n // Flush the last batch of rows.\n if (rowValues.length > 0) {\n yield {tag: 'backfill', ...backfill, rowValues, status};\n totalMsgs++;\n logFlushed();\n }\n\n yield {tag: 'backfill-completed', ...backfill, status};\n elapsed = (performance.now() - start).toFixed(3);\n lc.info?.(\n `Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes ` +\n `(${elapsed} ms)`,\n );\n}\n\n/*\n Creates (and drops) a replication slot in order to obtain a snapshot\n * that corresponds with a specific LSN. Sets the snapshot on the\n * TransactionPool and returns the watermark corresponding to the LSN.\n \n (Note that PG's other LSN-related functions are not scoped to a\n * transaction; this is the only way to get set a transaction at a specific\n * LSN.)\n /\nasync function setSnapshot(\n lc: LogContext,\n upstreamURI: string,\n tx: TransactionPool,\n slotNamePrefix: string,\n) {\n const replicationSession = pgClient(lc, upstreamURI, {\n ['fetch_types']: false, // Necessary for the streaming protocol\n connection: {replication: 'database'}, // https://www.postgresql.org/docs/current/protocol-replication.html\n });\n const tempSlot = `${slotNamePrefix}_bf_${Date.now()}`;\n try {\n const {snapshot_name: snapshot, consistent_point: lsn} =\n await createReplicationSlot(lc, replicationSession, tempSlot);\n\n await tx.processReadTask(sql =>\n sql.unsafe(`SET TRANSACTION SNAPSHOT '${snapshot}'`),\n );\n // Once the snapshot has been set, the replication session and slot can\n // be closed / dropped.\n await replicationSession.unsafe(`DROP_REPLICATION_SLOT \"${tempSlot}\"`);\n\n const watermark = toStateVersionString(lsn);\n lc.info?.(`Opened snapshot transaction at LSN ${lsn} (${watermark})`);\n return watermark;\n } catch (e) {\n // In the event of a failure, clean up the replication slot if created.\n await replicationSession.unsafe(\n /sql*/\n `SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name = '${tempSlot}'`,\n );\n lc.error?.(`Failed to create backfill snapshot`, e);\n throw e;\n } finally {\n await replicationSession.end();\n }\n}\n\nfunction validateSchema(\n tx: TransactionPool,\n publications: string[],\n bf: BackfillRequest,\n watermark: string,\n): Promise<{\n tableSpec: PublishedTableSpec;\n backfill: BackfillParams;\n}> {\n return tx.processReadTask(async sql => {\n const {tables} = await getPublicationInfo(sql, publications);\n const spec = tables.find(\n spec => spec.schema === bf.table.schema && spec.name === bf.table.name,\n );\n if (!spec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table has been renamed or dropped`,\n );\n }\n const tableMeta = v.parse(bf.table.metadata, tableMetadataSchema);\n if (spec.schemaOID !== tableMeta.schemaOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Schema no longer corresponds to the original schema`,\n );\n }\n if (spec.oid !== tableMeta.relationOID) {\n throw new SchemaIncompatibilityError(\n bf,\n `Table no longer corresponds to the original table`,\n );\n }\n if (\n !equals(\n new Set(Object.keys(tableMeta.rowKey)),\n new Set(spec.replicaIdentityColumns),\n )\n ) {\n throw new SchemaIncompatibilityError(\n bf,\n 'Row key (e.g. PRIMARY KEY or INDEX) has changed',\n );\n }\n const allCols = [\n ...Object.entries(tableMeta.rowKey),\n ...Object.entries(bf.columns),\n ];\n for (const [col, val] of allCols) {\n const colSpec = spec.columns[col];\n if (!colSpec) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} has been renamed or dropped`,\n );\n }\n const colMeta = v.parse(val, columnMetadataSchema);\n if (colMeta.attNum !== colSpec.pos) {\n throw new SchemaIncompatibilityError(\n bf,\n `Column ${col} no longer corresponds to the original column`,\n );\n }\n }\n const backfill: BackfillParams = {\n relation: {\n schema: bf.table.schema,\n name: bf.table.name,\n rowKey: {columns: Object.keys(tableMeta.rowKey)},\n },\n columns: Object.keys(bf.columns).filter(\n col => !(col in tableMeta.rowKey),\n ),\n watermark,\n };\n return {tableSpec: spec, backfill};\n });\n}\n"],"names":["spec","v.parse"],"mappings":";;;;;;;;;;;;;;AAgDA,MAAM,2BAA2B,KAAK;AAQtC,gBAAuB,eACrB,IACA,aACA,EAAC,MAAM,gBACP,IACA,OAAsB,IAC+B;AACrD,OAAK,GACF,YAAY,aAAa,UAAU,EACnC,YAAY,SAAS,GAAG,MAAM,IAAI;AAErC,QAAM,EAAC,sBAAsB,yBAAA,IAA4B;AACzD,QAAM,KAAK,SAAS,IAAI,aAAa;AAAA,IACnC,YAAY,EAAC,CAAC,kBAAkB,GAAG,kBAAA;AAAA,EAAiB,CACrD;AACD,QAAM,KAAK,IAAI,gBAAgB,IAAI,QAAQ,EAAE,IAAI,EAAE;AACnD,MAAI;AACF,UAAM,YAAY,MAAM,YAAY,IAAI,aAAa,IAAI,IAAI;AAC7D,UAAM,EAAC,WAAW,SAAA,IAAY,MAAM;AAAA,MAClC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,UAAM,QAAQ,MAAM,eAAe,IAAI,EAAC,oBAAoB,MAAK;AAGjE,UAAM,EAAC,UAAU,QAAA,IAAW;AAC5B,UAAM,OAAO,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,OAAO;AAEpD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,uBAAuB,WAAW,IAAI;AAAA,MACtC,KAAK,IAAI,CAAA,QAAO,MAAM,cAAc,UAAU,QAAQ,GAAG,EAAE,OAAO,CAAC;AAAA,MACnE;AAAA,IAAA;AAAA,EAEJ,SAAS,GAAG;AASV,QACE,aAAa,SAAS,kBACrB,EAAE,SAAS,sBAAsB,EAAE,SAAS,sBAC7C;AACA,YAAM,IAAI,2BAA2B,IAAI,OAAO,CAAC,GAAG,EAAC,OAAO,GAAE;AAAA,IAChE;AACA,UAAM;AAAA,EACR,UAAA;AACE,OAAG,QAAA;AAEH,SAAK,GAAG,OAAO,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7B,SAAK,GAAG,MAAM,MAAM,OAAK,GAAG,OAAO,qCAAqC,CAAC,CAAC;AAAA,EAC5E;AACF;AAEA,gBAAgB,OACd,IACA,IACA,UACA,EAAC,QAAQ,cAAc,cAAA,GACvB,YACA,qBACqD;AACrD,QAAM,QAAQ,YAAY,IAAA;AAC1B,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,GAAG;AAAA,IAAgB,CAAA,QAC7C,QAAQ,IAAI;AAAA,MACV,IAAI,OAA8B,YAAY;AAAA,MAC9C,IAAI,OAA+B,aAAa;AAAA,IAAA,CACjD;AAAA,EAAA;AAEH,QAAM,SAAyB;AAAA,IAC7B,MAAM;AAAA,IACN,WAAW,OAAO,KAAK,CAAC,EAAE,SAAS;AAAA,IACnC,YAAY,OAAO,MAAM,CAAC,EAAE,UAAU;AAAA,EAAA;AAGxC,MAAI,WAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AACnD,KAAG,OAAO,sCAAsC,MAAM,KAAK,OAAO,QAAQ;AAAA,IACxE;AAAA,EAAA,CACD;AACD,QAAM,aAAa,MAAM,GAAG;AAAA,IAAgB,SAC1C,IAAI,OAAO,SAAS,MAAM,aAAa,EAAE,SAAA;AAAA,EAAS;AAGpD,QAAM,YAAY,IAAI,UAAA;AACtB,MAAI,aAAa;AACjB,MAAI,YAAY;AAChB,MAAI,YAA2B,CAAA;AAC/B,MAAI,gBAAgB;AAEpB,QAAM,aAAa,MAAM;AACvB,OAAG;AAAA,MACD,WAAW,UAAU,MAAM,UAAU,aAAa,uBAChC,OAAO,IAAI,UAAU,SAAS,WAAW,UAAU;AAAA,IAAA;AAAA,EAEzE;AAGA,MAAI,MAAmB,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC7D,MAAI,MAAM;AAEV,mBAAiB,QAAQ,YAAY;AACnC,UAAM,QAAQ;AACd,eAAW,QAAQ,UAAU,MAAM,KAAK,GAAG;AACzC,UAAI,GAAG,IAAI,SAAS,OAAO,OAAQ,WAAW,GAAG,EAAE,IAAI;AAEvD,UAAI,EAAE,QAAQ,WAAW,QAAQ;AAC/B,kBAAU,KAAK,GAAG;AAClB,eAAO;AACP,cAAM,MAAM,KAAK,EAAC,QAAQ,WAAW,QAAO;AAC5C,cAAM;AAAA,MACR;AAAA,IACF;AACA,qBAAiB,MAAM;AACvB,kBAAc,MAAM;AAEpB,QAAI,iBAAiB,qBAAqB;AACxC,YAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,iBAAA;AACA,kBAAY,CAAA;AACZ,sBAAgB;AAAA,IAClB;AAAA,EACF;AAGA,MAAI,UAAU,SAAS,GAAG;AACxB,UAAM,EAAC,KAAK,YAAY,GAAG,UAAU,WAAW,OAAA;AAChD;AACA,eAAA;AAAA,EACF;AAEA,QAAM,EAAC,KAAK,sBAAsB,GAAG,UAAU,OAAA;AAC/C,aAAW,YAAY,IAAA,IAAQ,OAAO,QAAQ,CAAC;AAC/C,KAAG;AAAA,IACD,sBAAsB,OAAO,IAAI,UAAU,SAAS,UAAU,UAAU,WAClE,OAAO;AAAA,EAAA;AAEjB;AAWA,eAAe,YACb,IACA,aACA,IACA,gBACA;AACA,QAAM,qBAAqB,SAAS,IAAI,aAAa;AAAA,IACnD,CAAC,aAAa,GAAG;AAAA;AAAA,IACjB,YAAY,EAAC,aAAa,WAAA;AAAA;AAAA,EAAU,CACrC;AACD,QAAM,WAAW,GAAG,cAAc,OAAO,KAAK,KAAK;AACnD,MAAI;AACF,UAAM,EAAC,eAAe,UAAU,kBAAkB,QAChD,MAAM,sBAAsB,IAAI,oBAAoB,QAAQ;AAE9D,UAAM,GAAG;AAAA,MAAgB,CAAA,QACvB,IAAI,OAAO,6BAA6B,QAAQ,GAAG;AAAA,IAAA;AAIrD,UAAM,mBAAmB,OAAO,0BAA0B,QAAQ,GAAG;AAErE,UAAM,YAAY,qBAAqB,GAAG;AAC1C,OAAG,OAAO,sCAAsC,GAAG,KAAK,SAAS,GAAG;AACpE,WAAO;AAAA,EACT,SAAS,GAAG;AAEV,UAAM,mBAAmB;AAAA;AAAA,MAEvB;AAAA,8BACwB,QAAQ;AAAA,IAAA;AAElC,OAAG,QAAQ,sCAAsC,CAAC;AAClD,UAAM;AAAA,EACR,UAAA;AACE,UAAM,mBAAmB,IAAA;AAAA,EAC3B;AACF;AAEA,SAAS,eACP,IACA,cACA,IACA,WAIC;AACD,SAAO,GAAG,gBAAgB,OAAM,QAAO;AACrC,UAAM,EAAC,OAAA,IAAU,MAAM,mBAAmB,KAAK,YAAY;AAC3D,UAAM,OAAO,OAAO;AAAA,MAClB,CAAAA,UAAQA,MAAK,WAAW,GAAG,MAAM,UAAUA,MAAK,SAAS,GAAG,MAAM;AAAA,IAAA;AAEpE,QAAI,CAAC,MAAM;AACT,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,YAAYC,MAAQ,GAAG,MAAM,UAAU,mBAAmB;AAChE,QAAI,KAAK,cAAc,UAAU,WAAW;AAC1C,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QAAI,KAAK,QAAQ,UAAU,aAAa;AACtC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,QACE,CAAC;AAAA,MACC,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,CAAC;AAAA,MACrC,IAAI,IAAI,KAAK,sBAAsB;AAAA,IAAA,GAErC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM,UAAU;AAAA,MACd,GAAG,OAAO,QAAQ,UAAU,MAAM;AAAA,MAClC,GAAG,OAAO,QAAQ,GAAG,OAAO;AAAA,IAAA;AAE9B,eAAW,CAAC,KAAK,GAAG,KAAK,SAAS;AAChC,YAAM,UAAU,KAAK,QAAQ,GAAG;AAChC,UAAI,CAAC,SAAS;AACZ,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AACA,YAAM,UAAUA,MAAQ,KAAK,oBAAoB;AACjD,UAAI,QAAQ,WAAW,QAAQ,KAAK;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,UACA,UAAU,GAAG;AAAA,QAAA;AAAA,MAEjB;AAAA,IACF;AACA,UAAM,WAA2B;AAAA,MAC/B,UAAU;AAAA,QACR,QAAQ,GAAG,MAAM;AAAA,QACjB,MAAM,GAAG,MAAM;AAAA,QACf,QAAQ,EAAC,SAAS,OAAO,KAAK,UAAU,MAAM,EAAA;AAAA,MAAC;AAAA,MAEjD,SAAS,OAAO,KAAK,GAAG,OAAO,EAAE;AAAA,QAC/B,CAAA,QAAO,EAAE,OAAO,UAAU;AAAA,MAAA;AAAA,MAE5B;AAAA,IAAA;AAEF,WAAO,EAAC,WAAW,MAAM,SAAA;AAAA,EAC3B,CAAC;AACH;"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts CHANGED Viewed

@@ -19,6 +19,11 @@ type ReplicationSlot = {
 };
 export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, slotName: string): Promise<ReplicationSlot>;
 export declare const INSERT_BATCH_SIZE = 50;
-export declare function makeSelectPublishedStmt(table: PublishedTableSpec, columns: string[]): string;
+export type DownloadStatements = {
+    select: string;
+    getTotalRows: string;
+    getTotalBytes: string;
+};
+export declare function makeDownloadStatements(table: PublishedTableSpec, cols: string[]): DownloadStatements;
 export {};
 //# sourceMappingURL=initial-sync.d.ts.map

package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;~~AAGzE~~,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAa9D,OAAO,KAAK,EAAY,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAexE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAkB1D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,~~iBAyLvB~~;AAkGD,KAAK,eAAe,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAKF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAQ1B;AA6BD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,wBAAgB,~~uBAAuB~~,~~CACrC~~,KAAK,EAAE,kBAAkB,EACzB,~~OAAO~~,EAAE,MAAM,EAAE,~~UAYlB~~"}
1	+ {"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAIzE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAa9D,OAAO,KAAK,EAAY,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAexE,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAkB1D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACnC,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,iBAiMvB;AAkGD,KAAK,eAAe,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAKF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAQ1B;AA6BD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,kBAAkB,CAgBpB"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.js CHANGED Viewed

@@ -119,15 +119,23 @@ async function initialSync(lc, shard, tx, upstreamURI, syncOptions, context) {
     );
     try {
       createLiteTables(tx, tables, initialVersion);
+      const downloads = await Promise.all(
+        tables.map(
+          (spec) => copiers.processReadTask(
+            (db, lc2) => getInitialDownloadState(lc2, db, spec)
+          )
+        )
+      );
       statusPublisher.publish(
         lc,
         "Initializing",
         `Copying ${numTables} upstream tables at version ${initialVersion}`,
-        5e3
+        5e3,
+        () => ({ downloadStatus: downloads.map(({ status }) => status) })
       );
       void copyProfiler?.start();
       const rowCounts = await Promise.all(
-        tables.map(
+        downloads.map(
           (table) => copiers.processReadTask(
             (db, lc2) => copy(lc2, table, copyPool, db, tx)
           )
@@ -285,20 +293,58 @@ const INSERT_BATCH_SIZE = 50;
 const MB = 1024 * 1024;
 const MAX_BUFFERED_ROWS = 1e4;
 const BUFFERED_SIZE_THRESHOLD = 8 * MB;
-function makeSelectPublishedStmt(table, columns) {
+function makeDownloadStatements(table, cols) {
   const filterConditions = Object.values(table.publications).map(({ rowFilter }) => rowFilter).filter((f) => !!f);
-  return (
+  const where = filterConditions.length === 0 ? "" : (
     /*sql*/
-    `
-    SELECT ${columns.map(id).join(",")} FROM ${id(table.schema)}.${id(table.name)}` + (filterConditions.length === 0 ? "" : (
-      /*sql*/
-      ` WHERE ${filterConditions.join(" OR ")}`
-    ))
+    `WHERE ${filterConditions.join(" OR ")}`
   );
+  const fromTable = (
+    /*sql*/
+    `FROM ${id(table.schema)}.${id(table.name)} ${where}`
+  );
+  const totalBytes = `(${cols.map((col) => `SUM(COALESCE(pg_column_size(${id(col)}), 0))`).join(" + ")})`;
+  const stmts = {
+    select: (
+      /*sql*/
+      `SELECT ${cols.map(id).join(",")} ${fromTable}`
+    ),
+    getTotalRows: (
+      /*sql*/
+      `SELECT COUNT(*) AS "totalRows" ${fromTable}`
+    ),
+    getTotalBytes: (
+      /*sql*/
+      `SELECT ${totalBytes} AS "totalBytes" ${fromTable}`
+    )
+  };
+  return stmts;
+}
+async function getInitialDownloadState(lc, sql, spec) {
+  const start = performance.now();
+  const table = liteTableName(spec);
+  const columns = Object.keys(spec.columns);
+  const stmts = makeDownloadStatements(spec, columns);
+  const rowsResult = sql.unsafe(stmts.getTotalRows).execute();
+  const bytesResult = sql.unsafe(stmts.getTotalBytes).execute();
+  const state = {
+    spec,
+    status: {
+      table,
+      columns,
+      rows: 0,
+      totalRows: Number((await rowsResult)[0].totalRows),
+      totalBytes: Number((await bytesResult)[0].totalBytes)
+    }
+  };
+  const elapsed = (performance.now() - start).toFixed(3);
+  lc.info?.(`Computed initial download state for ${table} (${elapsed} ms)`, {
+    state: state.status
+  });
+  return state;
 }
-async function copy(lc, table, dbClient, from, to) {
+async function copy(lc, { spec: table, status }, dbClient, from, to) {
   const start = performance.now();
-  let rows = 0;
   let flushTime = 0;
   const tableName = liteTableName(table);
   const orderedColumns = Object.entries(table.columns);
@@ -315,7 +361,7 @@ async function copy(lc, table, dbClient, from, to) {
   const insertBatchStmt = to.prepare(
     insertSql + `,${valuesSql}`.repeat(INSERT_BATCH_SIZE - 1)
   );
-  const selectStmt = makeSelectPublishedStmt(table, columnNames);
+  const { select } = makeDownloadStatements(table, columnNames);
   const valuesPerRow = columnSpecs.length;
   const valuesPerBatch = valuesPerRow * INSERT_BATCH_SIZE;
   const pendingValues = Array.from({
@@ -338,14 +384,14 @@ async function copy(lc, table, dbClient, from, to) {
       pendingValues[i] = void 0;
     }
     pendingSize = 0;
-    rows += flushedRows;
+    status.rows += flushedRows;
     const elapsed2 = performance.now() - start2;
     flushTime += elapsed2;
     lc.debug?.(
       `flushed ${flushedRows} ${tableName} rows (${flushedSize} bytes) in ${elapsed2.toFixed(3)} ms`
     );
   }
-  lc.info?.(`Starting copy stream of ${tableName}:`, selectStmt);
+  lc.info?.(`Starting copy stream of ${tableName}:`, select);
   const pgParsers = await getTypeParsers(dbClient, { returnJsonAsString: true });
   const parsers = columnSpecs.map((c) => {
     const pgParse = pgParsers.getTypeParser(c.typeOID);
@@ -358,7 +404,7 @@ async function copy(lc, table, dbClient, from, to) {
   const tsvParser = new TsvParser();
   let col = 0;
   await pipeline(
-    await from.unsafe(`COPY (${selectStmt}) TO STDOUT`).readable(),
+    await from.unsafe(`COPY (${select}) TO STDOUT`).readable(),
     new Writable({
       highWaterMark: BUFFERED_SIZE_THRESHOLD,
       write(chunk, _encoding, callback) {
@@ -390,14 +436,14 @@ async function copy(lc, table, dbClient, from, to) {
   );
   const elapsed = performance.now() - start;
   lc.info?.(
-    `Finished copying ${rows} rows into ${tableName} (flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `
+    `Finished copying ${status.rows} rows into ${tableName} (flush: ${flushTime.toFixed(3)} ms) (total: ${elapsed.toFixed(3)} ms) `
   );
-  return { rows, flushTime };
+  return { rows: status.rows, flushTime };
 }
 export {
   INSERT_BATCH_SIZE,
   createReplicationSlot,
   initialSync,
-  makeSelectPublishedStmt
+  makeDownloadStatements
 };
 //# sourceMappingURL=initial-sync.js.map