@rocicorp/zero 1.5.0-canary.1 → 1.5.0-canary.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +1 -1
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +1 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +2 -9
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +7 -17
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/ivm/memory-source.js +1 -1
- package/out/zql/src/ivm/take.js +2 -2
- package/out/zqlite/src/table-source.js +1 -1
- package/package.json +1 -1
- package/out/zql/src/ivm/cap.d.ts +0 -32
- package/out/zql/src/ivm/cap.d.ts.map +0 -1
- package/out/zql/src/ivm/cap.js +0 -205
- package/out/zql/src/ivm/cap.js.map +0 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
var package_default = {
|
|
2
2
|
name: "@rocicorp/zero",
|
|
3
|
-
version: "1.5.0-canary.
|
|
3
|
+
version: "1.5.0-canary.3",
|
|
4
4
|
description: "Zero is a web framework for serverless web development.",
|
|
5
5
|
homepage: "https://zero.rocicorp.dev",
|
|
6
6
|
bugs: { "url": "https://bugs.rocicorp.dev" },
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.5.0-canary.
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.5.0-canary.3\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --type-aware src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.3.11\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.203.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.62.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.203.0\",\n \"@opentelemetry/resources\": \"^2.0.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.0.1\",\n \"@opentelemetry/sdk-node\": \"^0.203.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^5.4.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.0.17\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.2.0\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"js-xxhash\": \"^4.0.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"oxfmt\": \"^0.45.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.5.4\",\n \"tsx\": \"^4.21.0\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"@vitest/runner\": \"4.1.3\",\n \"analyze-query\": \"0.0.0\",\n \"ast-to-zql\": \"0.0.0\",\n \"expo-sqlite\": \">=15\",\n \"replicache\": \"15.2.1\",\n \"shared\": \"0.0.0\",\n \"syncpack\": \"^14.3.0\",\n \"typedoc\": \"^0.28.17\",\n \"typedoc-plugin-markdown\": \"^4.10.0\",\n \"typescript\": \"~6.0.2\",\n \"vite\": \"8.0.3\",\n \"vitest\": \"4.1.3\",\n \"zero-cache\": \"0.0.0\",\n \"zero-client\": \"0.0.0\",\n \"zero-pg\": \"0.0.0\",\n \"zero-protocol\": \"0.0.0\",\n \"zero-react\": \"0.0.0\",\n \"zero-server\": \"0.0.0\",\n \"zero-solid\": \"0.0.0\",\n \"zqlite\": \"0.0.0\"\n },\n \"peerDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"expo-sqlite\": \">=15\",\n \"kysely\": \"^0.28.16\"\n },\n \"peerDependenciesMeta\": {\n \"kysely\": {\n \"optional\": true\n },\n \"expo-sqlite\": {\n \"optional\": true\n },\n \"@op-engineering/op-sqlite\": {\n \"optional\": true\n }\n },\n \"engines\": {\n \"node\": \">=22\"\n }\n}"],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA6BA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AA6BA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAuMf"}
|
|
@@ -77,7 +77,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
77
77
|
await upgradeReplica(lc, "change-streamer-init", replica.file);
|
|
78
78
|
const { backupURL, port: metricsPort } = litestream;
|
|
79
79
|
const monitor = backupURL ? new BackupMonitor(lc, replica.file, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, Date.now() - workerStartTime) : new ReplicaMonitor(lc, replica.file, changeStreamer);
|
|
80
|
-
const changeStreamerWebServer = new ChangeStreamerHttpServer(lc,
|
|
80
|
+
const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
|
|
81
81
|
port,
|
|
82
82
|
keepaliveTimeoutMs,
|
|
83
83
|
startupDelayMs
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n BackupNotFoundException,\n restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const workerStartTime = Date.now();\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n },\n upstream,\n change,\n replica,\n initialSync,\n litestream,\n keepaliveTimeoutMs,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n const lc = createLogContext(config, 'change-streamer');\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = pgClient(\n lc,\n change.db,\n 'change-streamer',\n {\n max: change.maxConns,\n },\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const {autoReset, replicationLag} = config;\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date, then acquire\n // a lock to prevent change-lock purges. This ensures that (this)\n // change-streamer will be able to resume from the backup.\n await initChangeStreamerSchema(lc, changeDB, shard);\n let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n // Restore from litestream if the change-log has entries.\n if (purgeLock) {\n try {\n await restoreReplica(lc, config, purgeLock);\n } catch (e) {\n // If the restore failed, e.g. due to a corrupt or missing backup, the\n // replication-manager recovers by re-syncing.\n const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n lc[log]?.(\n `error restoring backup. resyncing the replica: ${String(e)}`,\n e,\n );\n\n // The purgeLock must be released if the backup could not be restored,\n // or it will otherwise prevent the change-db update after the resync\n // completes.\n await purgeLock.release();\n purgeLock = null;\n }\n }\n\n let changeStreamer: ChangeStreamerService | undefined;\n\n const context = getServerContext(config);\n\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {changeSource, subscriptionState} =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n statementTimeoutMs: change.statementTimeoutMs,\n },\n setTimeout,\n );\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n // Perform any upgrades to the replica in case it was restored from an\n // earlier version. Note that this upgrade is done by the replicator worker\n // as well (in both the replication-manager and the view-syncer), but the\n // change-streamer independently reads the replica, and it is fine run the\n // upgrade logic redundantly since it is idempotent.\n await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n const {backupURL, port: metricsPort} = litestream;\n const monitor = backupURL\n ? new BackupMonitor(\n lc,\n replica.file,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n changeStreamer,\n // The time between when the zero-cache was started to when the\n // change-streamer is ready to start serves as the initial delay for\n // watermark cleanup (as it either includes a similar replica\n // restoration/preparation step, or an initial-sync, which\n // generally takes longer).\n //\n // Consider: Also account for permanent volumes?\n Date.now() - workerStartTime,\n )\n : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n config,\n {port, keepaliveTimeoutMs, startupDelayMs},\n parent,\n changeStreamer,\n monitor instanceof BackupMonitor ? monitor : null,\n );\n\n parent.send(['ready', {ready: true}]);\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer.\n return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,KAAK;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;CACnD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,YACA,uBACE;AAEJ,eACE,iBAAiB,QAAQ,mBAAmB,GAAG,MAAM,EACrD,mBACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,kBAAkB;AACtD,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP,mBACA,EACE,KAAK,OAAO,UACb,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;AAKpC,OAAM,yBAAyB,IAAI,UAAU,MAAM;CACnD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,SAAS,CAAC,SAAS;AAGpE,KAAI,UACF,KAAI;AACF,QAAM,eAAe,IAAI,QAAQ,UAAU;UACpC,GAAG;AAIV,KADY,aAAa,0BAA0B,SAAS,WAE1D,kDAAkD,OAAO,EAAE,IAC3D,EACD;AAKD,QAAM,UAAU,SAAS;AACzB,cAAY;;CAIhB,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;GAC5B,EACD,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAY1B,SAAM,WAAW,SAAS;AAC1B,eAAY;AACZ;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,gBACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAoB;EAAe,EAC1C,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
1
|
+
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n BackupNotFoundException,\n restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const workerStartTime = Date.now();\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n },\n upstream,\n change,\n replica,\n initialSync,\n litestream,\n keepaliveTimeoutMs,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n const lc = createLogContext(config, 'change-streamer');\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = pgClient(\n lc,\n change.db,\n 'change-streamer',\n {\n max: change.maxConns,\n },\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const {autoReset, replicationLag} = config;\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date, then acquire\n // a lock to prevent change-lock purges. This ensures that (this)\n // change-streamer will be able to resume from the backup.\n await initChangeStreamerSchema(lc, changeDB, shard);\n let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n // Restore from litestream if the change-log has entries.\n if (purgeLock) {\n try {\n await restoreReplica(lc, config, purgeLock);\n } catch (e) {\n // If the restore failed, e.g. due to a corrupt or missing backup, the\n // replication-manager recovers by re-syncing.\n const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n lc[log]?.(\n `error restoring backup. resyncing the replica: ${String(e)}`,\n e,\n );\n\n // The purgeLock must be released if the backup could not be restored,\n // or it will otherwise prevent the change-db update after the resync\n // completes.\n await purgeLock.release();\n purgeLock = null;\n }\n }\n\n let changeStreamer: ChangeStreamerService | undefined;\n\n const context = getServerContext(config);\n\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {changeSource, subscriptionState} =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n statementTimeoutMs: change.statementTimeoutMs,\n },\n setTimeout,\n );\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n // Perform any upgrades to the replica in case it was restored from an\n // earlier version. Note that this upgrade is done by the replicator worker\n // as well (in both the replication-manager and the view-syncer), but the\n // change-streamer independently reads the replica, and it is fine run the\n // upgrade logic redundantly since it is idempotent.\n await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n const {backupURL, port: metricsPort} = litestream;\n const monitor = backupURL\n ? new BackupMonitor(\n lc,\n replica.file,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n changeStreamer,\n // The time between when the zero-cache was started to when the\n // change-streamer is ready to start serves as the initial delay for\n // watermark cleanup (as it either includes a similar replica\n // restoration/preparation step, or an initial-sync, which\n // generally takes longer).\n //\n // Consider: Also account for permanent volumes?\n Date.now() - workerStartTime,\n )\n : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs},\n parent,\n changeStreamer,\n monitor instanceof BackupMonitor ? monitor : null,\n );\n\n parent.send(['ready', {ready: true}]);\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer.\n return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,KAAK;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;CACnD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,YACA,uBACE;AAEJ,eACE,iBAAiB,QAAQ,mBAAmB,GAAG,MAAM,EACrD,mBACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,kBAAkB;AACtD,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP,mBACA,EACE,KAAK,OAAO,UACb,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;AAKpC,OAAM,yBAAyB,IAAI,UAAU,MAAM;CACnD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,SAAS,CAAC,SAAS;AAGpE,KAAI,UACF,KAAI;AACF,QAAM,eAAe,IAAI,QAAQ,UAAU;UACpC,GAAG;AAIV,KADY,aAAa,0BAA0B,SAAS,WAE1D,kDAAkD,OAAO,EAAE,IAC3D,EACD;AAKD,QAAM,UAAU,SAAS;AACzB,cAAY;;CAIhB,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;GAC5B,EACD,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAY1B,SAAM,WAAW,SAAS;AAC1B,eAAY;AACZ;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,gBACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAe,EAC1C,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { IncomingMessage } from 'node:http';
|
|
2
2
|
import type { LogContext } from '@rocicorp/logger';
|
|
3
|
-
import type { ZeroConfig } from '../../config/zero-config.ts';
|
|
4
3
|
import { type Worker } from '../../types/processes.ts';
|
|
5
4
|
import { type ShardID } from '../../types/shards.ts';
|
|
6
5
|
import { type Source } from '../../types/streams.ts';
|
|
@@ -17,7 +16,7 @@ type Options = {
|
|
|
17
16
|
export declare class ChangeStreamerHttpServer extends HttpService {
|
|
18
17
|
#private;
|
|
19
18
|
readonly id = "change-streamer-http-server";
|
|
20
|
-
constructor(lc: LogContext,
|
|
19
|
+
constructor(lc: LogContext, opts: Options, parent: Worker, changeStreamer: ChangeStreamer & Service, backupMonitor: BackupMonitor | null);
|
|
21
20
|
protected _onStart(): void;
|
|
22
21
|
protected _onStop(): Promise<void>;
|
|
23
22
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAsB,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAIxE,OAAO,EAAC,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAC/C,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAwB,KAAK,eAAe,EAAC,MAAM,eAAe,CAAC;AAW1E,KAAK,OAAO,GAAG;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;gBAO1C,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,cAAc,GAAG,OAAO,EACxC,aAAa,EAAE,aAAa,GAAG,IAAI;cA4GlB,QAAQ,IAAI,IAAI;cAWV,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAKlD;AAED,qBAAa,wBAAyB,YAAW,cAAc;;gBAO3D,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS;IA8BjC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IASjE,SAAS,CAAC,GAAG,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;CAQrE;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAc3E"}
|
|
@@ -25,16 +25,9 @@ var ChangeStreamerHttpServer = class extends HttpService {
|
|
|
25
25
|
#opts;
|
|
26
26
|
#changeStreamer;
|
|
27
27
|
#backupMonitor;
|
|
28
|
-
constructor(lc,
|
|
28
|
+
constructor(lc, opts, parent, changeStreamer, backupMonitor) {
|
|
29
29
|
super("change-streamer-http-server", lc, opts, async (fastify) => {
|
|
30
|
-
|
|
31
|
-
if (config.websocketCompression) if (config.websocketCompressionOptions) try {
|
|
32
|
-
websocketOptions.perMessageDeflate = JSON.parse(config.websocketCompressionOptions);
|
|
33
|
-
} catch (e) {
|
|
34
|
-
throw new Error(`Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`);
|
|
35
|
-
}
|
|
36
|
-
else websocketOptions.perMessageDeflate = true;
|
|
37
|
-
await fastify.register(websocket, { options: websocketOptions });
|
|
30
|
+
await fastify.register(websocket);
|
|
38
31
|
fastify.get(CHANGES_PATH_PATTERN, { websocket: true }, this.#subscribe);
|
|
39
32
|
fastify.get(SNAPSHOT_PATH_PATTERN, { websocket: true }, this.#reserveSnapshot);
|
|
40
33
|
installWebSocketReceiver(lc, fastify.websocketServer, this.#receiveWebsocket, parent);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n downstreamSchema,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n port: number;\n keepaliveTimeoutMs: number | undefined;\n startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n readonly id = 'change-streamer-http-server';\n readonly #lc: LogContext;\n readonly #opts: Options;\n readonly #changeStreamer: ChangeStreamer & Service;\n readonly #backupMonitor: BackupMonitor | null;\n\n constructor(\n lc: LogContext,\n config: ZeroConfig,\n opts: Options,\n parent: Worker,\n changeStreamer: ChangeStreamer & Service,\n backupMonitor: BackupMonitor | null,\n ) {\n super('change-streamer-http-server', lc, opts, async fastify => {\n const websocketOptions: {perMessageDeflate?: boolean | object} = {};\n if (config.websocketCompression) {\n if (config.websocketCompressionOptions) {\n try {\n websocketOptions.perMessageDeflate = JSON.parse(\n config.websocketCompressionOptions,\n );\n } catch (e) {\n throw new Error(\n `Failed to parse ZERO_WEBSOCKET_COMPRESSION_OPTIONS: ${String(e)}. Expected valid JSON.`,\n );\n }\n } else {\n websocketOptions.perMessageDeflate = true;\n }\n }\n\n await fastify.register(websocket, {\n options: websocketOptions,\n });\n\n fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n fastify.get(\n SNAPSHOT_PATH_PATTERN,\n {websocket: true},\n this.#reserveSnapshot,\n );\n\n installWebSocketReceiver<'snapshot' | 'changes'>(\n lc,\n fastify.websocketServer,\n this.#receiveWebsocket,\n parent,\n );\n });\n\n this.#lc = lc;\n this.#opts = opts;\n this.#changeStreamer = changeStreamer;\n this.#backupMonitor = backupMonitor;\n }\n\n #getBackupMonitor() {\n return must(\n this.#backupMonitor,\n 'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n );\n }\n\n // Called when receiving a web socket via the main dispatcher handoff.\n readonly #receiveWebsocket = (\n ws: WebSocket,\n action: 'changes' | 'snapshot',\n msg: IncomingMessageSubset,\n ) => {\n switch (action) {\n case 'snapshot':\n return this.#reserveSnapshot(ws, msg);\n case 'changes':\n return this.#subscribe(ws, msg);\n default:\n closeWithError(\n this._lc,\n ws,\n `invalid action \"${action}\" received in handoff`,\n );\n return;\n }\n };\n\n readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n checkProtocolVersion(url.pathname);\n const taskID = url.searchParams.get('taskID');\n if (!taskID) {\n throw new Error('Missing taskID in snapshot request');\n }\n const downstream =\n this.#getBackupMonitor().startSnapshotReservation(taskID);\n void streamOut(this._lc, downstream, ws);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n try {\n const ctx = getSubscriberContext(req);\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n\n const downstream = await this.#changeStreamer.subscribe(ctx);\n if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n // Now that the change-streamer knows about the subscriber and watermark,\n // end the reservation to safely resume scheduling cleanup.\n this.#backupMonitor.endReservation(ctx.taskID);\n }\n void streamOut(this._lc, downstream, ws);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n #changeStreamerStarted = false;\n\n #ensureChangeStreamerStarted(reason: string) {\n if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n void this.#changeStreamer\n .run()\n .catch(e =>\n this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n )\n .finally(() => this.stop());\n\n this.#changeStreamerStarted = true;\n }\n }\n\n protected override _onStart(): void {\n const {startupDelayMs} = this.#opts;\n this._state.setTimeout(\n () =>\n this.#ensureChangeStreamerStarted(\n `startup delay elapsed (${startupDelayMs} ms)`,\n ),\n startupDelayMs,\n );\n }\n\n protected override async _onStop(): Promise<void> {\n if (this.#changeStreamerStarted) {\n await this.#changeStreamer.stop();\n }\n }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n readonly #lc: LogContext;\n readonly #shardID: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #changeStreamerURI: string | undefined;\n\n constructor(\n lc: LogContext,\n shardID: ShardID,\n changeDB: string,\n changeStreamerURI: string | undefined,\n ) {\n this.#lc = lc;\n this.#shardID = shardID;\n // Create a pg client with a single short-lived connection for the purpose\n // of change-streamer discovery (i.e. ChangeDB as DNS).\n this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n max: 1,\n ['idle_timeout']: 15,\n });\n this.#changeStreamerURI = changeStreamerURI;\n }\n\n async #resolveChangeStreamer(path: string) {\n let baseURL = this.#changeStreamerURI;\n if (!baseURL) {\n const address = await discoverChangeStreamerAddress(\n this.#shardID,\n this.#changeDB,\n );\n if (!address) {\n throw new Error(`no change-streamer is running`);\n }\n baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n }\n const uri = new URL(path, baseURL);\n this.#lc.info?.(`connecting to change-streamer@${uri}`);\n return uri;\n }\n\n async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n const params = new URLSearchParams({taskID});\n const ws = new WebSocket(uri + `?${params.toString()}`);\n\n return streamIn(this.#lc, ws, snapshotMessageSchema);\n }\n\n async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n const params = getParams(ctx);\n const ws = new WebSocket(uri + `?${params.toString()}`);\n\n return streamIn(this.#lc, ws, downstreamSchema);\n }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n const protocolVersion = checkProtocolVersion(url.pathname);\n const params = new URLParams(url);\n\n return {\n protocolVersion,\n id: params.get('id', true),\n taskID: params.get('taskID', false),\n mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n replicaVersion: params.get('replicaVersion', true),\n watermark: params.get('watermark', true),\n initial: params.getBoolean('initial'),\n };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n const match = PATH_REGEX.exec(pathname);\n if (!match) {\n throw new Error(`invalid path: ${pathname}`);\n }\n const v = Number(match.groups?.version);\n if (\n Number.isNaN(v) ||\n v > PROTOCOL_VERSION ||\n v < MIN_SUPPORTED_PROTOCOL_VERSION\n ) {\n throw new Error(\n `Cannot service client at protocol v${v}. ` +\n `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n );\n }\n return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n // The protocolVersion is hard-coded into the CHANGES_PATH.\n const {protocolVersion, ...stringParams} = ctx;\n assert(\n protocolVersion === PROTOCOL_VERSION,\n `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n );\n return new URLSearchParams({\n ...stringParams,\n taskID: ctx.taskID ? ctx.taskID : '',\n initial: ctx.initial ? 'true' : 'false',\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,mBAA2D,EAAE;AACnE,OAAI,OAAO,qBACT,KAAI,OAAO,4BACT,KAAI;AACF,qBAAiB,oBAAoB,KAAK,MACxC,OAAO,4BACR;YACM,GAAG;AACV,UAAM,IAAI,MACR,uDAAuD,OAAO,EAAE,CAAC,wBAClE;;OAGH,kBAAiB,oBAAoB;AAIzC,SAAM,QAAQ,SAAS,WAAW,EAChC,SAAS,kBACV,CAAC;AAEF,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}
|
|
1
|
+
{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {streamIn, streamOut, type Source} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {Service} from '../service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n downstreamSchema,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n port: number;\n keepaliveTimeoutMs: number | undefined;\n startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n readonly id = 'change-streamer-http-server';\n readonly #lc: LogContext;\n readonly #opts: Options;\n readonly #changeStreamer: ChangeStreamer & Service;\n readonly #backupMonitor: BackupMonitor | null;\n\n constructor(\n lc: LogContext,\n opts: Options,\n parent: Worker,\n changeStreamer: ChangeStreamer & Service,\n backupMonitor: BackupMonitor | null,\n ) {\n super('change-streamer-http-server', lc, opts, async fastify => {\n await fastify.register(websocket);\n\n fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n fastify.get(\n SNAPSHOT_PATH_PATTERN,\n {websocket: true},\n this.#reserveSnapshot,\n );\n\n installWebSocketReceiver<'snapshot' | 'changes'>(\n lc,\n fastify.websocketServer,\n this.#receiveWebsocket,\n parent,\n );\n });\n\n this.#lc = lc;\n this.#opts = opts;\n this.#changeStreamer = changeStreamer;\n this.#backupMonitor = backupMonitor;\n }\n\n #getBackupMonitor() {\n return must(\n this.#backupMonitor,\n 'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n );\n }\n\n // Called when receiving a web socket via the main dispatcher handoff.\n readonly #receiveWebsocket = (\n ws: WebSocket,\n action: 'changes' | 'snapshot',\n msg: IncomingMessageSubset,\n ) => {\n switch (action) {\n case 'snapshot':\n return this.#reserveSnapshot(ws, msg);\n case 'changes':\n return this.#subscribe(ws, msg);\n default:\n closeWithError(\n this._lc,\n ws,\n `invalid action \"${action}\" received in handoff`,\n );\n return;\n }\n };\n\n readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n try {\n const url = new URL(\n req.url ?? '',\n req.headers.origin ?? 'http://localhost',\n );\n checkProtocolVersion(url.pathname);\n const taskID = url.searchParams.get('taskID');\n if (!taskID) {\n throw new Error('Missing taskID in snapshot request');\n }\n const downstream =\n this.#getBackupMonitor().startSnapshotReservation(taskID);\n void streamOut(this._lc, downstream, ws);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n try {\n const ctx = getSubscriberContext(req);\n if (ctx.mode === 'serving') {\n this.#ensureChangeStreamerStarted('incoming subscription');\n }\n\n const downstream = await this.#changeStreamer.subscribe(ctx);\n if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n // Now that the change-streamer knows about the subscriber and watermark,\n // end the reservation to safely resume scheduling cleanup.\n this.#backupMonitor.endReservation(ctx.taskID);\n }\n void streamOut(this._lc, downstream, ws);\n } catch (err) {\n closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n }\n };\n\n #changeStreamerStarted = false;\n\n #ensureChangeStreamerStarted(reason: string) {\n if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n void this.#changeStreamer\n .run()\n .catch(e =>\n this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n )\n .finally(() => this.stop());\n\n this.#changeStreamerStarted = true;\n }\n }\n\n protected override _onStart(): void {\n const {startupDelayMs} = this.#opts;\n this._state.setTimeout(\n () =>\n this.#ensureChangeStreamerStarted(\n `startup delay elapsed (${startupDelayMs} ms)`,\n ),\n startupDelayMs,\n );\n }\n\n protected override async _onStop(): Promise<void> {\n if (this.#changeStreamerStarted) {\n await this.#changeStreamer.stop();\n }\n }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n readonly #lc: LogContext;\n readonly #shardID: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #changeStreamerURI: string | undefined;\n\n constructor(\n lc: LogContext,\n shardID: ShardID,\n changeDB: string,\n changeStreamerURI: string | undefined,\n ) {\n this.#lc = lc;\n this.#shardID = shardID;\n // Create a pg client with a single short-lived connection for the purpose\n // of change-streamer discovery (i.e. ChangeDB as DNS).\n this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n max: 1,\n ['idle_timeout']: 15,\n });\n this.#changeStreamerURI = changeStreamerURI;\n }\n\n async #resolveChangeStreamer(path: string) {\n let baseURL = this.#changeStreamerURI;\n if (!baseURL) {\n const address = await discoverChangeStreamerAddress(\n this.#shardID,\n this.#changeDB,\n );\n if (!address) {\n throw new Error(`no change-streamer is running`);\n }\n baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n }\n const uri = new URL(path, baseURL);\n this.#lc.info?.(`connecting to change-streamer@${uri}`);\n return uri;\n }\n\n async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n const params = new URLSearchParams({taskID});\n const ws = new WebSocket(uri + `?${params.toString()}`);\n\n return streamIn(this.#lc, ws, snapshotMessageSchema);\n }\n\n async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n const params = getParams(ctx);\n const ws = new WebSocket(uri + `?${params.toString()}`);\n\n return streamIn(this.#lc, ws, downstreamSchema);\n }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n const protocolVersion = checkProtocolVersion(url.pathname);\n const params = new URLParams(url);\n\n return {\n protocolVersion,\n id: params.get('id', true),\n taskID: params.get('taskID', false),\n mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n replicaVersion: params.get('replicaVersion', true),\n watermark: params.get('watermark', true),\n initial: params.getBoolean('initial'),\n };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n const match = PATH_REGEX.exec(pathname);\n if (!match) {\n throw new Error(`invalid path: ${pathname}`);\n }\n const v = Number(match.groups?.version);\n if (\n Number.isNaN(v) ||\n v > PROTOCOL_VERSION ||\n v < MIN_SUPPORTED_PROTOCOL_VERSION\n ) {\n throw new Error(\n `Cannot service client at protocol v${v}. ` +\n `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n );\n }\n return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n // The protocolVersion is hard-coded into the CHANGES_PATH.\n const {protocolVersion, ...stringParams} = ctx;\n assert(\n protocolVersion === PROTOCOL_VERSION,\n `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n );\n return new URLSearchParams({\n ...stringParams,\n taskID: ctx.taskID ? ctx.taskID : '',\n initial: ctx.initial ? 'true' : 'false',\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AA2BA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;AACA,QAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;AAC9D,SAAM,QAAQ,SAAS,UAAU;AAEjC,WAAQ,IAAI,sBAAsB,EAAC,WAAW,MAAK,EAAE,MAAA,UAAgB;AACrE,WAAQ,IACN,uBACA,EAAC,WAAW,MAAK,EACjB,MAAA,gBACD;AAED,4BACE,IACA,QAAQ,iBACR,MAAA,kBACA,OACD;IACD;AAEF,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,iBAAuB;AACvB,QAAA,gBAAsB;;CAGxB,oBAAoB;AAClB,SAAO,KACL,MAAA,eACA,0EACD;;CAIH,qBACE,IACA,QACA,QACG;AACH,UAAQ,QAAR;GACE,KAAK,WACH,QAAO,MAAA,gBAAsB,IAAI,IAAI;GACvC,KAAK,UACH,QAAO,MAAA,UAAgB,IAAI,IAAI;GACjC;AACE,mBACE,KAAK,KACL,IACA,mBAAmB,OAAO,uBAC3B;AACD;;;CAIN,oBAA6B,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,mBACvB;AACD,wBAAqB,IAAI,SAAS;GAClC,MAAM,SAAS,IAAI,aAAa,IAAI,SAAS;AAC7C,OAAI,CAAC,OACH,OAAM,IAAI,MAAM,qCAAqC;GAEvD,MAAM,aACJ,MAAA,kBAAwB,CAAC,yBAAyB,OAAO;AACtD,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,aAAsB,OAAO,IAAe,QAAwB;AAClE,MAAI;GACF,MAAM,MAAM,qBAAqB,IAAI;AACrC,OAAI,IAAI,SAAS,UACf,OAAA,4BAAkC,wBAAwB;GAG5D,MAAM,aAAa,MAAM,MAAA,eAAqB,UAAU,IAAI;AAC5D,OAAI,IAAI,WAAW,IAAI,UAAU,MAAA,cAG/B,OAAA,cAAoB,eAAe,IAAI,OAAO;AAE3C,aAAU,KAAK,KAAK,YAAY,GAAG;WACjC,KAAK;AACZ,kBAAe,KAAK,KAAK,IAAI,KAAK,eAAe;;;CAIrD,yBAAyB;CAEzB,6BAA6B,QAAgB;AAC3C,MAAI,CAAC,MAAA,yBAA+B,KAAK,OAAO,WAAW,EAAE;AAC3D,SAAA,GAAS,OAAO,mCAAmC,SAAS;AACvD,SAAA,eACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,OAAO,0CAA0C,EAAE,CAC7D,CACA,cAAc,KAAK,MAAM,CAAC;AAE7B,SAAA,wBAA8B;;;CAIlC,WAAoC;EAClC,MAAM,EAAC,mBAAkB,MAAA;AACzB,OAAK,OAAO,iBAER,MAAA,4BACE,0BAA0B,eAAe,MAC1C,EACH,eACD;;CAGH,MAAyB,UAAyB;AAChD,MAAI,MAAA,sBACF,OAAM,MAAA,eAAqB,MAAM;;;AAKvC,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;AACA,QAAA,KAAW;AACX,QAAA,UAAgB;AAGhB,QAAA,WAAiB,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;GACnB,CAAC;AACF,QAAA,oBAA0B;;CAG5B,OAAA,sBAA6B,MAAc;EACzC,IAAI,UAAU,MAAA;AACd,MAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,MAAA,SACA,MAAA,SACD;AACD,OAAI,CAAC,QACH,OAAM,IAAI,MAAM,gCAAgC;AAElD,aAAU,QAAQ,SAAS,MAAM,GAAG,GAAG,QAAQ,KAAK,QAAQ,QAAQ;;EAEtE,MAAM,MAAM,IAAI,IAAI,MAAM,QAAQ;AAClC,QAAA,GAAS,OAAO,iCAAiC,MAAM;AACvD,SAAO;;CAGT,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,cAAc,GAG7B,IADhB,IAAI,gBAAgB,EAAC,QAAO,CAAC,CACF,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,sBAAsB;;CAGtD,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAHH,MAAM,MAAA,sBAA4B,aAAa,GAG5B,IADhB,UAAU,IAAI,CACa,UAAU,GAAG;AAEvD,SAAO,SAAS,MAAA,IAAU,IAAI,iBAAiB;;;AAMnD,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,mBAAmB;CAC5E,MAAM,kBAAkB,qBAAqB,IAAI,SAAS;CAC1D,MAAM,SAAS,IAAI,UAAU,IAAI;AAEjC,QAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,KAAK;EAC1B,QAAQ,OAAO,IAAI,UAAU,MAAM;EACnC,MAAM,OAAO,IAAI,QAAQ,MAAM,KAAK,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,KAAK;EAClD,WAAW,OAAO,IAAI,aAAa,KAAK;EACxC,SAAS,OAAO,WAAW,UAAU;EACtC;;AAGH,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,SAAS;AACvC,KAAI,CAAC,MACH,OAAM,IAAI,MAAM,iBAAiB,WAAW;CAE9C,MAAM,IAAI,OAAO,MAAM,QAAQ,QAAQ;AACvC,KACE,OAAO,MAAM,EAAE,IACf,IAAA,KACA,IAAI,+BAEJ,OAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,UAC5D;AAEH,QAAO;;AAIT,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;AAC3C,QACE,oBAAA,GACA,oDACD;AACD,QAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;EACjC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../../../../zql/src/builder/builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,6BAA6B,CAAC;AAE3D,OAAO,KAAK,EACV,GAAG,EAGH,SAAS,EAIT,WAAW,EAEX,QAAQ,EAIT,MAAM,mCAAmC,CAAC;AAC3C,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,oCAAoC,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,2CAA2C,CAAC;
|
|
1
|
+
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../../../../zql/src/builder/builder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,6BAA6B,CAAC;AAE3D,OAAO,KAAK,EACV,GAAG,EAGH,SAAS,EAIT,WAAW,EAEX,QAAQ,EAIT,MAAM,mCAAmC,CAAC;AAC3C,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,oCAAoC,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,2CAA2C,CAAC;AAI1E,OAAO,EAEL,KAAK,WAAW,EACjB,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAC,MAAM,oBAAoB,CAAC;AAElE,OAAO,KAAK,EAAC,MAAM,EAAE,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAK1D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,kCAAkC,CAAC;AAC1E,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAE9D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAkB,KAAK,mBAAmB,EAAC,MAAM,aAAa,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClD,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAElC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAE/C;;;;OAIG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAEjD;;;OAGG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAErC,aAAa,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAEjD,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;IAElD,mBAAmB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAEnE,mBAAmB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;IAEhE;;;;;OAKG;IACH,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,GAAG,EACR,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,mBAAmB,EAC/B,EAAE,CAAC,EAAE,UAAU,EACf,YAAY,CAAC,EAAE,YAAY,GAC1B,KAAK,CAWP;AAED,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,GAAG,EACR,qBAAqB,EAAE,qBAAqB,GAAG,SAAS,OAsDzD;AAyBD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,CAsB5D;AAqQD,wBAAgB,OAAO,CACrB,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,WAAW,EACtB,QAAQ,EAAE,eAAe,EACzB,IAAI,EAAE,MAAM,GACX,WAAW,CAsCb;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,WAAW,6EAa7D;AAED,wBAAgB,8BAA8B,CAC5C,SAAS,EAAE,SAAS,GACnB,SAAS,IAAI,mBAAmB,CAQlC;AAsHD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,QAAQ,EAClB,EAAE,EAAE,UAAU,GACb,IAAI,CAgBN;AA8CD,wBAAgB,0CAA0C,CACxD,IAAI,EAAE,SAAS,GACd,OAAO,CAWT;AAED,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,SAAS,SAAS,EAAE,EAChC,SAAS,EAAE,CAAC,CAAC,EAAE,SAAS,KAAK,OAAO,uCAYrC"}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
import { assert, unreachable } from "../../../shared/src/asserts.js";
|
|
2
2
|
import { must } from "../../../shared/src/must.js";
|
|
3
|
-
import { completeOrdering } from "../query/complete-ordering.js";
|
|
4
|
-
import { Take } from "../ivm/take.js";
|
|
5
|
-
import { Cap } from "../ivm/cap.js";
|
|
6
3
|
import { buildFilterPipeline } from "../ivm/filter-operators.js";
|
|
7
4
|
import { Exists } from "../ivm/exists.js";
|
|
8
5
|
import { FanIn } from "../ivm/fan-in.js";
|
|
@@ -11,6 +8,8 @@ import { Filter } from "../ivm/filter.js";
|
|
|
11
8
|
import { FlippedJoin } from "../ivm/flipped-join.js";
|
|
12
9
|
import { Join } from "../ivm/join.js";
|
|
13
10
|
import { Skip } from "../ivm/skip.js";
|
|
11
|
+
import { completeOrdering } from "../query/complete-ordering.js";
|
|
12
|
+
import { Take } from "../ivm/take.js";
|
|
14
13
|
import { UnionFanIn } from "../ivm/union-fan-in.js";
|
|
15
14
|
import { UnionFanOut } from "../ivm/union-fan-out.js";
|
|
16
15
|
import { planQuery } from "../planner/planner-builder.js";
|
|
@@ -113,7 +112,7 @@ function assertNoNotExists(condition) {
|
|
|
113
112
|
default: unreachable(condition);
|
|
114
113
|
}
|
|
115
114
|
}
|
|
116
|
-
function buildPipelineInternal(ast, delegate, queryID, name, partitionKey
|
|
115
|
+
function buildPipelineInternal(ast, delegate, queryID, name, partitionKey) {
|
|
117
116
|
const source = delegate.getSource(ast.table);
|
|
118
117
|
if (!source) throw new Error(`Source not found: ${ast.table}`);
|
|
119
118
|
ast = uniquifyCorrelatedSubqueryConditionAliases(ast);
|
|
@@ -126,11 +125,7 @@ function buildPipelineInternal(ast, delegate, queryID, name, partitionKey, isNon
|
|
|
126
125
|
for (const key of csq.related.correlation.parentField) splitEditKeys.add(key);
|
|
127
126
|
}
|
|
128
127
|
if (ast.related) for (const csq of ast.related) for (const key of csq.correlation.parentField) splitEditKeys.add(key);
|
|
129
|
-
|
|
130
|
-
assert(ast.start === void 0, "EXISTS subqueries must not have start");
|
|
131
|
-
assert(ast.related === void 0, "EXISTS subqueries must not have related");
|
|
132
|
-
}
|
|
133
|
-
const conn = source.connect(isNonFlippedExistsChild ? void 0 : must(ast.orderBy), ast.where, splitEditKeys, delegate.debug);
|
|
128
|
+
const conn = source.connect(must(ast.orderBy), ast.where, splitEditKeys, delegate.debug);
|
|
134
129
|
let end = delegate.decorateSourceInput(conn, queryID);
|
|
135
130
|
end = delegate.decorateInput(end, `${name}:source(${ast.table})`);
|
|
136
131
|
const { fullyAppliedFilters } = conn;
|
|
@@ -147,12 +142,7 @@ function buildPipelineInternal(ast, delegate, queryID, name, partitionKey, isNon
|
|
|
147
142
|
}
|
|
148
143
|
}, delegate, queryID, end, name, true);
|
|
149
144
|
if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) end = applyWhere(end, ast.where, delegate, name);
|
|
150
|
-
if (ast.limit !== void 0)
|
|
151
|
-
const capName = `${name}:cap`;
|
|
152
|
-
const cap = new Cap(end, delegate.createStorage(capName), ast.limit, partitionKey);
|
|
153
|
-
delegate.addEdge(end, cap);
|
|
154
|
-
end = delegate.decorateInput(cap, capName);
|
|
155
|
-
} else {
|
|
145
|
+
if (ast.limit !== void 0) {
|
|
156
146
|
const takeName = `${name}:take`;
|
|
157
147
|
const take = new Take(end, delegate.createStorage(takeName), ast.limit, partitionKey);
|
|
158
148
|
delegate.addEdge(end, take);
|
|
@@ -202,7 +192,7 @@ function applyFilterWithFlips(input, condition, delegate, name) {
|
|
|
202
192
|
}
|
|
203
193
|
case "correlatedSubquery": {
|
|
204
194
|
const sq = condition.related;
|
|
205
|
-
const child = buildPipelineInternal(sq.subquery, delegate, "", `${name}.${sq.subquery.alias}`, sq.correlation.childField
|
|
195
|
+
const child = buildPipelineInternal(sq.subquery, delegate, "", `${name}.${sq.subquery.alias}`, sq.correlation.childField);
|
|
206
196
|
const flippedJoin = new FlippedJoin({
|
|
207
197
|
parent: end,
|
|
208
198
|
child,
|
|
@@ -285,7 +275,7 @@ function valuePosName(left) {
|
|
|
285
275
|
function applyCorrelatedSubQuery(sq, delegate, queryID, end, name, fromCondition) {
|
|
286
276
|
if (sq.subquery.limit === 0 && fromCondition) return end;
|
|
287
277
|
assert(sq.subquery.alias, "Subquery must have an alias");
|
|
288
|
-
const child = buildPipelineInternal(sq.subquery, delegate, queryID, `${name}.${sq.subquery.alias}`, sq.correlation.childField
|
|
278
|
+
const child = buildPipelineInternal(sq.subquery, delegate, queryID, `${name}.${sq.subquery.alias}`, sq.correlation.childField);
|
|
289
279
|
const joinName = `${name}:join(${sq.subquery.alias})`;
|
|
290
280
|
const join = new Join({
|
|
291
281
|
parent: end,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.js","names":[],"sources":["../../../../../zql/src/builder/builder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport type {JSONValue} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport type {\n AST,\n ColumnReference,\n CompoundKey,\n Condition,\n Conjunction,\n CorrelatedSubquery,\n CorrelatedSubqueryCondition,\n Disjunction,\n LiteralValue,\n Ordering,\n Parameter,\n SimpleCondition,\n ValuePosition,\n} from '../../../zero-protocol/src/ast.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts';\nimport {Cap} from '../ivm/cap.ts';\nimport {Exists} from '../ivm/exists.ts';\nimport {FanIn} from '../ivm/fan-in.ts';\nimport {FanOut} from '../ivm/fan-out.ts';\nimport {\n buildFilterPipeline,\n type FilterInput,\n} from '../ivm/filter-operators.ts';\nimport {Filter} from '../ivm/filter.ts';\nimport {FlippedJoin} from '../ivm/flipped-join.ts';\nimport {Join} from '../ivm/join.ts';\nimport type {Input, InputBase, Storage} from '../ivm/operator.ts';\nimport {Skip} from '../ivm/skip.ts';\nimport type {Source, SourceInput} from '../ivm/source.ts';\nimport {Take} from '../ivm/take.ts';\nimport {UnionFanIn} from '../ivm/union-fan-in.ts';\nimport {UnionFanOut} from '../ivm/union-fan-out.ts';\nimport {planQuery} from '../planner/planner-builder.ts';\nimport type {ConnectionCostModel} from '../planner/planner-connection.ts';\nimport type {PlanDebugger} from '../planner/planner-debug.ts';\nimport {completeOrdering} from '../query/complete-ordering.ts';\nimport type {DebugDelegate} from './debug-delegate.ts';\nimport {createPredicate, type NoSubqueryCondition} from './filter.ts';\n\nexport type StaticQueryParameters = {\n authData: Record<string, JSONValue>;\n preMutationRow?: Row | undefined;\n};\n\n/**\n * Interface required of caller to buildPipeline. Connects to constructed\n * pipeline to delegate environment to provide sources and storage.\n */\nexport interface BuilderDelegate {\n readonly applyFiltersAnyway?: boolean | undefined;\n debug?: DebugDelegate | undefined;\n\n /**\n * When true, allows NOT EXISTS conditions in queries.\n * Defaults to false.\n *\n * We only set this to true on the server.\n * The client-side query engine cannot support NOT EXISTS because:\n * 1. Zero only syncs a subset of data to the client\n * 2. On the client, we can't distinguish between a row not existing vs.\n * a row not being synced to the client\n * 3. NOT EXISTS requires complete knowledge of what doesn't exist\n */\n readonly enableNotExists?: boolean | undefined;\n\n /**\n * Called once for each source needed by the AST.\n * Might be called multiple times with same tableName. It is OK to return\n * same storage instance in that case.\n */\n getSource(tableName: string): Source | undefined;\n\n /**\n * Called once for each operator that requires storage. Should return a new\n * unique storage object for each call.\n */\n createStorage(name: string): Storage;\n\n decorateInput(input: Input, name: string): Input;\n\n addEdge(source: InputBase, dest: InputBase): void;\n\n decorateFilterInput(input: FilterInput, name: string): FilterInput;\n\n decorateSourceInput(input: SourceInput, queryID: string): Input;\n\n /**\n * The AST is mapped on-the-wire between client and server names.\n *\n * There is no \"wire\" for zqlite tests so this function is provided\n * to allow tests to remap the AST.\n */\n mapAst?: ((ast: AST) => AST) | undefined;\n}\n\n/**\n * Builds a pipeline from an AST. Caller must provide a delegate to create source\n * and storage interfaces as necessary.\n *\n * Usage:\n *\n * ```ts\n * class MySink implements Output {\n * readonly #input: Input;\n *\n * constructor(input: Input) {\n * this.#input = input;\n * input.setOutput(this);\n * }\n *\n * push(change: Change, _: Operator) {\n * console.log(change);\n * }\n * }\n *\n * const input = buildPipeline(ast, myDelegate, hash(ast));\n * const sink = new MySink(input);\n * ```\n */\nexport function buildPipeline(\n ast: AST,\n delegate: BuilderDelegate,\n queryID: string,\n costModel?: ConnectionCostModel,\n lc?: LogContext,\n planDebugger?: PlanDebugger,\n): Input {\n ast = delegate.mapAst ? delegate.mapAst(ast) : ast;\n ast = completeOrdering(\n ast,\n tableName => must(delegate.getSource(tableName)).tableSchema.primaryKey,\n );\n\n if (costModel) {\n ast = planQuery(ast, costModel, planDebugger, lc);\n }\n return buildPipelineInternal(ast, delegate, queryID, '');\n}\n\nexport function bindStaticParameters(\n ast: AST,\n staticQueryParameters: StaticQueryParameters | undefined,\n) {\n const visit = (node: AST): AST => ({\n ...node,\n where: node.where ? bindCondition(node.where) : undefined,\n related: node.related?.map(sq => ({\n ...sq,\n subquery: visit(sq.subquery),\n })),\n });\n\n function bindCondition(condition: Condition): Condition {\n if (condition.type === 'simple') {\n return {\n ...condition,\n left: bindValue(condition.left),\n right: bindValue(condition.right) as Exclude<\n ValuePosition,\n ColumnReference\n >,\n };\n }\n if (condition.type === 'correlatedSubquery') {\n return {\n ...condition,\n related: {\n ...condition.related,\n subquery: visit(condition.related.subquery),\n },\n };\n }\n\n return {\n ...condition,\n conditions: condition.conditions.map(bindCondition),\n };\n }\n\n const bindValue = (value: ValuePosition): ValuePosition => {\n if (isParameter(value)) {\n const anchor = must(\n staticQueryParameters,\n 'Static query params do not exist',\n )[value.anchor];\n const resolvedValue = resolveField(anchor, value.field);\n return {\n type: 'literal',\n value: resolvedValue as LiteralValue,\n };\n }\n return value;\n };\n\n return visit(ast);\n}\n\nfunction resolveField(\n anchor: Record<string, JSONValue> | Row | undefined,\n field: string | string[],\n): unknown {\n if (anchor === undefined) {\n return null;\n }\n\n if (Array.isArray(field)) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n return field.reduce((acc, f) => (acc as any)?.[f], anchor) ?? null;\n }\n\n return anchor[field] ?? null;\n}\n\nfunction isParameter(value: ValuePosition): value is Parameter {\n return value.type === 'static';\n}\n\nconst EXISTS_LIMIT = 3;\nconst PERMISSIONS_EXISTS_LIMIT = 1;\n\n/**\n * Checks if a condition tree contains any NOT EXISTS operations.\n * Recursively checks AND/OR branches but does not recurse into nested subqueries\n * (those are checked when buildPipelineInternal processes them).\n */\nexport function assertNoNotExists(condition: Condition): void {\n switch (condition.type) {\n case 'simple':\n return;\n\n case 'correlatedSubquery':\n if (condition.op === 'NOT EXISTS') {\n throw new Error(\n 'not(exists()) is not supported on the client - see https://bugs.rocicorp.dev/issue/3438',\n );\n }\n return;\n\n case 'and':\n case 'or':\n for (const c of condition.conditions) {\n assertNoNotExists(c);\n }\n return;\n default:\n unreachable(condition);\n }\n}\n\nfunction buildPipelineInternal(\n ast: AST,\n delegate: BuilderDelegate,\n queryID: string,\n name: string,\n partitionKey?: CompoundKey,\n isNonFlippedExistsChild?: boolean | undefined,\n): Input {\n const source = delegate.getSource(ast.table);\n if (!source) {\n throw new Error(`Source not found: ${ast.table}`);\n }\n\n ast = uniquifyCorrelatedSubqueryConditionAliases(ast);\n\n if (!delegate.enableNotExists && ast.where) {\n assertNoNotExists(ast.where);\n }\n\n const csqConditions = gatherCorrelatedSubqueryQueryConditions(ast.where);\n const splitEditKeys: Set<string> = partitionKey\n ? new Set(partitionKey)\n : new Set();\n const aliases = new Set<string>();\n for (const csq of csqConditions) {\n aliases.add(csq.related.subquery.alias || '');\n for (const key of csq.related.correlation.parentField) {\n splitEditKeys.add(key);\n }\n }\n if (ast.related) {\n for (const csq of ast.related) {\n for (const key of csq.correlation.parentField) {\n splitEditKeys.add(key);\n }\n }\n }\n if (isNonFlippedExistsChild) {\n assert(ast.start === undefined, 'EXISTS subqueries must not have start');\n assert(\n ast.related === undefined,\n 'EXISTS subqueries must not have related',\n );\n }\n\n const conn = source.connect(\n // exists pipelines are unordered — orderBy is ignored here.\n // Non-exists pipelines always have orderBy completed with PKs.\n isNonFlippedExistsChild ? undefined : must(ast.orderBy),\n ast.where,\n splitEditKeys,\n delegate.debug,\n );\n\n let end: Input = delegate.decorateSourceInput(conn, queryID);\n end = delegate.decorateInput(end, `${name}:source(${ast.table})`);\n const {fullyAppliedFilters} = conn;\n\n if (ast.start) {\n const skip = new Skip(end, ast.start);\n delegate.addEdge(end, skip);\n end = delegate.decorateInput(skip, `${name}:skip)`);\n }\n\n for (const csqCondition of csqConditions) {\n // flipped EXISTS are handled in applyWhere\n if (!csqCondition.flip) {\n end = applyCorrelatedSubQuery(\n {\n ...csqCondition.related,\n subquery: {\n ...csqCondition.related.subquery,\n limit:\n csqCondition.related.system === 'permissions'\n ? PERMISSIONS_EXISTS_LIMIT\n : EXISTS_LIMIT,\n },\n },\n delegate,\n queryID,\n end,\n name,\n true,\n );\n }\n }\n\n if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) {\n end = applyWhere(end, ast.where, delegate, name);\n }\n\n if (ast.limit !== undefined) {\n // We end `exists` pipelines with `cap`\n // The reason is that `cap` does not care about the order of the pipeline.\n // This allows SQLite to chose the order and never end up creating temp b-trees.\n // The problem with SQLite creating a temp b-tree is it will incur a scan of the entire\n // result set where exists only needs the first row.\n if (isNonFlippedExistsChild) {\n const capName = `${name}:cap`;\n const cap = new Cap(\n end,\n delegate.createStorage(capName),\n ast.limit,\n partitionKey,\n );\n delegate.addEdge(end, cap);\n end = delegate.decorateInput(cap, capName);\n } else {\n const takeName = `${name}:take`;\n const take = new Take(\n end,\n delegate.createStorage(takeName),\n ast.limit,\n partitionKey,\n );\n delegate.addEdge(end, take);\n end = delegate.decorateInput(take, takeName);\n }\n }\n\n if (ast.related) {\n // Dedupe by alias - last one wins (LWW), like limit(5).limit(10)\n const byAlias = new Map<string, CorrelatedSubquery>();\n for (const csq of ast.related) {\n byAlias.set(csq.subquery.alias ?? '', csq);\n }\n for (const csq of byAlias.values()) {\n end = applyCorrelatedSubQuery(csq, delegate, queryID, end, name, false);\n }\n }\n\n return end;\n}\n\nfunction applyWhere(\n input: Input,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): Input {\n if (!conditionIncludesFlippedSubqueryAtAnyLevel(condition)) {\n return buildFilterPipeline(input, delegate, filterInput =>\n applyFilter(filterInput, condition, delegate, name),\n );\n }\n\n return applyFilterWithFlips(input, condition, delegate, name);\n}\n\nfunction applyFilterWithFlips(\n input: Input,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): Input {\n let end = input;\n assert(condition.type !== 'simple', 'Simple conditions cannot have flips');\n\n switch (condition.type) {\n case 'and': {\n const [withFlipped, withoutFlipped] = partitionBranches(\n condition.conditions,\n conditionIncludesFlippedSubqueryAtAnyLevel,\n );\n if (withoutFlipped.length > 0) {\n end = buildFilterPipeline(input, delegate, filterInput =>\n applyAnd(\n filterInput,\n {\n type: 'and',\n conditions: withoutFlipped,\n },\n delegate,\n name,\n ),\n );\n }\n assert(withFlipped.length > 0, 'Impossible to have no flips here');\n for (const cond of withFlipped) {\n end = applyFilterWithFlips(end, cond, delegate, name);\n }\n break;\n }\n case 'or': {\n const [withFlipped, withoutFlipped] = partitionBranches(\n condition.conditions,\n conditionIncludesFlippedSubqueryAtAnyLevel,\n );\n assert(withFlipped.length > 0, 'Impossible to have no flips here');\n\n const ufo = new UnionFanOut(end);\n delegate.addEdge(end, ufo);\n end = delegate.decorateInput(ufo, `${name}:ufo`);\n\n const branches: Input[] = [];\n if (withoutFlipped.length > 0) {\n branches.push(\n buildFilterPipeline(end, delegate, filterInput =>\n applyOr(\n filterInput,\n {\n type: 'or',\n conditions: withoutFlipped,\n },\n delegate,\n name,\n ),\n ),\n );\n }\n\n for (const cond of withFlipped) {\n branches.push(applyFilterWithFlips(end, cond, delegate, name));\n }\n\n const ufi = new UnionFanIn(ufo, branches);\n for (const branch of branches) {\n delegate.addEdge(branch, ufi);\n }\n end = delegate.decorateInput(ufi, `${name}:ufi`);\n\n break;\n }\n case 'correlatedSubquery': {\n const sq = condition.related;\n const child = buildPipelineInternal(\n sq.subquery,\n delegate,\n '',\n `${name}.${sq.subquery.alias}`,\n sq.correlation.childField,\n false,\n );\n const flippedJoin = new FlippedJoin({\n parent: end,\n child,\n parentKey: sq.correlation.parentField,\n childKey: sq.correlation.childField,\n relationshipName: must(\n sq.subquery.alias,\n 'Subquery must have an alias',\n ),\n hidden: sq.hidden ?? false,\n system: sq.system ?? 'client',\n });\n delegate.addEdge(end, flippedJoin);\n delegate.addEdge(child, flippedJoin);\n end = delegate.decorateInput(\n flippedJoin,\n `${name}:flipped-join(${sq.subquery.alias})`,\n );\n break;\n }\n }\n\n return end;\n}\n\nfunction applyFilter(\n input: FilterInput,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n switch (condition.type) {\n case 'and':\n return applyAnd(input, condition, delegate, name);\n case 'or':\n return applyOr(input, condition, delegate, name);\n case 'correlatedSubquery':\n return applyCorrelatedSubqueryCondition(input, condition, delegate, name);\n case 'simple':\n return applySimpleCondition(input, delegate, condition);\n }\n}\n\nfunction applyAnd(\n input: FilterInput,\n condition: Conjunction,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n for (const subCondition of condition.conditions) {\n input = applyFilter(input, subCondition, delegate, name);\n }\n return input;\n}\n\nexport function applyOr(\n input: FilterInput,\n condition: Disjunction,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n const [subqueryConditions, otherConditions] =\n groupSubqueryConditions(condition);\n // if there are no subquery conditions, no fan-in / fan-out is needed\n if (subqueryConditions.length === 0) {\n const filter = new Filter(\n input,\n createPredicate({\n type: 'or',\n conditions: otherConditions,\n }),\n );\n delegate.addEdge(input, filter);\n return filter;\n }\n\n const fanOut = new FanOut(input);\n delegate.addEdge(input, fanOut);\n const branches = subqueryConditions.map(subCondition =>\n applyFilter(fanOut, subCondition, delegate, name),\n );\n if (otherConditions.length > 0) {\n const filter = new Filter(\n fanOut,\n createPredicate({\n type: 'or',\n conditions: otherConditions,\n }),\n );\n delegate.addEdge(fanOut, filter);\n branches.push(filter);\n }\n const ret = new FanIn(fanOut, branches);\n for (const branch of branches) {\n delegate.addEdge(branch, ret);\n }\n fanOut.setFanIn(ret);\n return ret;\n}\n\nexport function groupSubqueryConditions(condition: Disjunction) {\n const partitioned: [\n subqueryConditions: Condition[],\n otherConditions: NoSubqueryCondition[],\n ] = [[], []];\n for (const subCondition of condition.conditions) {\n if (isNotAndDoesNotContainSubquery(subCondition)) {\n partitioned[1].push(subCondition);\n } else {\n partitioned[0].push(subCondition);\n }\n }\n return partitioned;\n}\n\nexport function isNotAndDoesNotContainSubquery(\n condition: Condition,\n): condition is NoSubqueryCondition {\n if (condition.type === 'correlatedSubquery') {\n return false;\n }\n if (condition.type === 'simple') {\n return true;\n }\n return condition.conditions.every(isNotAndDoesNotContainSubquery);\n}\n\nfunction applySimpleCondition(\n input: FilterInput,\n delegate: BuilderDelegate,\n condition: SimpleCondition,\n): FilterInput {\n const filter = new Filter(input, createPredicate(condition));\n delegate.decorateFilterInput(\n filter,\n `${valuePosName(condition.left)}:${condition.op}:${valuePosName(condition.right)}`,\n );\n delegate.addEdge(input, filter);\n return filter;\n}\n\nfunction valuePosName(left: ValuePosition) {\n switch (left.type) {\n case 'static':\n return left.field;\n case 'literal':\n return left.value;\n case 'column':\n return left.name;\n }\n}\n\nfunction applyCorrelatedSubQuery(\n sq: CorrelatedSubquery,\n delegate: BuilderDelegate,\n queryID: string,\n end: Input,\n name: string,\n fromCondition: boolean,\n) {\n // TODO: we only omit the join if the CSQ if from a condition since\n // we want to create an empty array for `related` fields that are `limit(0)`\n if (sq.subquery.limit === 0 && fromCondition) {\n return end;\n }\n\n assert(sq.subquery.alias, 'Subquery must have an alias');\n const child = buildPipelineInternal(\n sq.subquery,\n delegate,\n queryID,\n `${name}.${sq.subquery.alias}`,\n sq.correlation.childField,\n fromCondition,\n );\n\n const joinName = `${name}:join(${sq.subquery.alias})`;\n const join = new Join({\n parent: end,\n child,\n parentKey: sq.correlation.parentField,\n childKey: sq.correlation.childField,\n relationshipName: sq.subquery.alias,\n hidden: sq.hidden ?? false,\n system: sq.system ?? 'client',\n });\n delegate.addEdge(end, join);\n delegate.addEdge(child, join);\n return delegate.decorateInput(join, joinName);\n}\n\nfunction applyCorrelatedSubqueryCondition(\n input: FilterInput,\n condition: CorrelatedSubqueryCondition,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n assert(\n condition.op === 'EXISTS' || condition.op === 'NOT EXISTS',\n 'Expected EXISTS or NOT EXISTS operator',\n );\n if (condition.related.subquery.limit === 0) {\n if (condition.op === 'EXISTS') {\n const filter = new Filter(input, () => false);\n delegate.addEdge(input, filter);\n return filter;\n }\n const filter = new Filter(input, () => true);\n delegate.addEdge(input, filter);\n return filter;\n }\n const existsName = `${name}:exists(${condition.related.subquery.alias})`;\n const exists = new Exists(\n input,\n must(condition.related.subquery.alias),\n condition.related.correlation.parentField,\n condition.op,\n );\n delegate.addEdge(input, exists);\n return delegate.decorateFilterInput(exists, existsName);\n}\n\nfunction gatherCorrelatedSubqueryQueryConditions(\n condition: Condition | undefined,\n) {\n const csqs: CorrelatedSubqueryCondition[] = [];\n const gather = (condition: Condition) => {\n if (condition.type === 'correlatedSubquery') {\n csqs.push(condition);\n return;\n }\n if (condition.type === 'and' || condition.type === 'or') {\n for (const c of condition.conditions) {\n gather(c);\n }\n return;\n }\n };\n if (condition) {\n gather(condition);\n }\n return csqs;\n}\n\nexport function assertOrderingIncludesPK(\n ordering: Ordering,\n pk: PrimaryKey,\n): void {\n // oxlint-disable-next-line unicorn/prefer-set-has -- Array is more appropriate here for small collections\n const orderingFields = ordering.map(([field]) => field);\n const missingFields = pk.filter(pkField => !orderingFields.includes(pkField));\n\n if (missingFields.length > 0) {\n throw new Error(\n `Ordering must include all primary key fields. Missing: ${missingFields.join(\n ', ',\n )}. ZQL automatically appends primary key fields to the ordering if they are missing \n so a common cause of this error is a casing mismatch between Postgres and ZQL.\n E.g., \"userid\" vs \"userID\".\n You may want to add double-quotes around your Postgres column names to prevent Postgres from lower-casing them:\n https://www.postgresql.org/docs/current/sql-syntax-lexical.htm`,\n );\n }\n}\n\nfunction uniquifyCorrelatedSubqueryConditionAliases(ast: AST): AST {\n if (!ast.where) {\n return ast;\n }\n const {where} = ast;\n if (where.type !== 'and' && where.type !== 'or') {\n return ast;\n }\n\n let count = 0;\n const uniquifyCorrelatedSubquery = (csqc: CorrelatedSubqueryCondition) => ({\n ...csqc,\n related: {\n ...csqc.related,\n subquery: {\n ...csqc.related.subquery,\n alias: (csqc.related.subquery.alias ?? '') + '_' + count++,\n },\n },\n });\n\n const uniquify = (cond: Condition): Condition => {\n if (cond.type === 'simple') {\n return cond;\n } else if (cond.type === 'correlatedSubquery') {\n return uniquifyCorrelatedSubquery(cond);\n }\n const conditions = [];\n for (const c of cond.conditions) {\n conditions.push(uniquify(c));\n }\n return {\n type: cond.type,\n conditions,\n };\n };\n\n const result = {\n ...ast,\n where: uniquify(where),\n };\n return result;\n}\n\nexport function conditionIncludesFlippedSubqueryAtAnyLevel(\n cond: Condition,\n): boolean {\n if (cond.type === 'correlatedSubquery') {\n return !!cond.flip;\n }\n if (cond.type === 'and' || cond.type === 'or') {\n return cond.conditions.some(c =>\n conditionIncludesFlippedSubqueryAtAnyLevel(c),\n );\n }\n // simple conditions don't have flips\n return false;\n}\n\nexport function partitionBranches(\n conditions: readonly Condition[],\n predicate: (c: Condition) => boolean,\n) {\n const matched: Condition[] = [];\n const notMatched: Condition[] = [];\n for (const c of conditions) {\n if (predicate(c)) {\n matched.push(c);\n } else {\n notMatched.push(c);\n }\n }\n return [matched, notMatched] as const;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6HA,SAAgB,cACd,KACA,UACA,SACA,WACA,IACA,cACO;AACP,OAAM,SAAS,SAAS,SAAS,OAAO,IAAI,GAAG;AAC/C,OAAM,iBACJ,MACA,cAAa,KAAK,SAAS,UAAU,UAAU,CAAC,CAAC,YAAY,WAC9D;AAED,KAAI,UACF,OAAM,UAAU,KAAK,WAAW,cAAc,GAAG;AAEnD,QAAO,sBAAsB,KAAK,UAAU,SAAS,GAAG;;AAG1D,SAAgB,qBACd,KACA,uBACA;CACA,MAAM,SAAS,UAAoB;EACjC,GAAG;EACH,OAAO,KAAK,QAAQ,cAAc,KAAK,MAAM,GAAG,KAAA;EAChD,SAAS,KAAK,SAAS,KAAI,QAAO;GAChC,GAAG;GACH,UAAU,MAAM,GAAG,SAAS;GAC7B,EAAE;EACJ;CAED,SAAS,cAAc,WAAiC;AACtD,MAAI,UAAU,SAAS,SACrB,QAAO;GACL,GAAG;GACH,MAAM,UAAU,UAAU,KAAK;GAC/B,OAAO,UAAU,UAAU,MAAM;GAIlC;AAEH,MAAI,UAAU,SAAS,qBACrB,QAAO;GACL,GAAG;GACH,SAAS;IACP,GAAG,UAAU;IACb,UAAU,MAAM,UAAU,QAAQ,SAAS;IAC5C;GACF;AAGH,SAAO;GACL,GAAG;GACH,YAAY,UAAU,WAAW,IAAI,cAAc;GACpD;;CAGH,MAAM,aAAa,UAAwC;AACzD,MAAI,YAAY,MAAM,EAAE;GACtB,MAAM,SAAS,KACb,uBACA,mCACD,CAAC,MAAM;AAER,UAAO;IACL,MAAM;IACN,OAHoB,aAAa,QAAQ,MAAM,MAAM;IAItD;;AAEH,SAAO;;AAGT,QAAO,MAAM,IAAI;;AAGnB,SAAS,aACP,QACA,OACS;AACT,KAAI,WAAW,KAAA,EACb,QAAO;AAGT,KAAI,MAAM,QAAQ,MAAM,CAEtB,QAAO,MAAM,QAAQ,KAAK,MAAO,MAAc,IAAI,OAAO,IAAI;AAGhE,QAAO,OAAO,UAAU;;AAG1B,SAAS,YAAY,OAA0C;AAC7D,QAAO,MAAM,SAAS;;AAGxB,IAAM,eAAe;AACrB,IAAM,2BAA2B;;;;;;AAOjC,SAAgB,kBAAkB,WAA4B;AAC5D,SAAQ,UAAU,MAAlB;EACE,KAAK,SACH;EAEF,KAAK;AACH,OAAI,UAAU,OAAO,aACnB,OAAM,IAAI,MACR,0FACD;AAEH;EAEF,KAAK;EACL,KAAK;AACH,QAAK,MAAM,KAAK,UAAU,WACxB,mBAAkB,EAAE;AAEtB;EACF,QACE,aAAY,UAAU;;;AAI5B,SAAS,sBACP,KACA,UACA,SACA,MACA,cACA,yBACO;CACP,MAAM,SAAS,SAAS,UAAU,IAAI,MAAM;AAC5C,KAAI,CAAC,OACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,QAAQ;AAGnD,OAAM,2CAA2C,IAAI;AAErD,KAAI,CAAC,SAAS,mBAAmB,IAAI,MACnC,mBAAkB,IAAI,MAAM;CAG9B,MAAM,gBAAgB,wCAAwC,IAAI,MAAM;CACxE,MAAM,gBAA6B,eAC/B,IAAI,IAAI,aAAa,mBACrB,IAAI,KAAK;CACb,MAAM,0BAAU,IAAI,KAAa;AACjC,MAAK,MAAM,OAAO,eAAe;AAC/B,UAAQ,IAAI,IAAI,QAAQ,SAAS,SAAS,GAAG;AAC7C,OAAK,MAAM,OAAO,IAAI,QAAQ,YAAY,YACxC,eAAc,IAAI,IAAI;;AAG1B,KAAI,IAAI,QACN,MAAK,MAAM,OAAO,IAAI,QACpB,MAAK,MAAM,OAAO,IAAI,YAAY,YAChC,eAAc,IAAI,IAAI;AAI5B,KAAI,yBAAyB;AAC3B,SAAO,IAAI,UAAU,KAAA,GAAW,wCAAwC;AACxE,SACE,IAAI,YAAY,KAAA,GAChB,0CACD;;CAGH,MAAM,OAAO,OAAO,QAGlB,0BAA0B,KAAA,IAAY,KAAK,IAAI,QAAQ,EACvD,IAAI,OACJ,eACA,SAAS,MACV;CAED,IAAI,MAAa,SAAS,oBAAoB,MAAM,QAAQ;AAC5D,OAAM,SAAS,cAAc,KAAK,GAAG,KAAK,UAAU,IAAI,MAAM,GAAG;CACjE,MAAM,EAAC,wBAAuB;AAE9B,KAAI,IAAI,OAAO;EACb,MAAM,OAAO,IAAI,KAAK,KAAK,IAAI,MAAM;AACrC,WAAS,QAAQ,KAAK,KAAK;AAC3B,QAAM,SAAS,cAAc,MAAM,GAAG,KAAK,QAAQ;;AAGrD,MAAK,MAAM,gBAAgB,cAEzB,KAAI,CAAC,aAAa,KAChB,OAAM,wBACJ;EACE,GAAG,aAAa;EAChB,UAAU;GACR,GAAG,aAAa,QAAQ;GACxB,OACE,aAAa,QAAQ,WAAW,gBAC5B,2BACA;GACP;EACF,EACD,UACA,SACA,KACA,MACA,KACD;AAIL,KAAI,IAAI,UAAU,CAAC,uBAAuB,SAAS,oBACjD,OAAM,WAAW,KAAK,IAAI,OAAO,UAAU,KAAK;AAGlD,KAAI,IAAI,UAAU,KAAA,EAMhB,KAAI,yBAAyB;EAC3B,MAAM,UAAU,GAAG,KAAK;EACxB,MAAM,MAAM,IAAI,IACd,KACA,SAAS,cAAc,QAAQ,EAC/B,IAAI,OACJ,aACD;AACD,WAAS,QAAQ,KAAK,IAAI;AAC1B,QAAM,SAAS,cAAc,KAAK,QAAQ;QACrC;EACL,MAAM,WAAW,GAAG,KAAK;EACzB,MAAM,OAAO,IAAI,KACf,KACA,SAAS,cAAc,SAAS,EAChC,IAAI,OACJ,aACD;AACD,WAAS,QAAQ,KAAK,KAAK;AAC3B,QAAM,SAAS,cAAc,MAAM,SAAS;;AAIhD,KAAI,IAAI,SAAS;EAEf,MAAM,0BAAU,IAAI,KAAiC;AACrD,OAAK,MAAM,OAAO,IAAI,QACpB,SAAQ,IAAI,IAAI,SAAS,SAAS,IAAI,IAAI;AAE5C,OAAK,MAAM,OAAO,QAAQ,QAAQ,CAChC,OAAM,wBAAwB,KAAK,UAAU,SAAS,KAAK,MAAM,MAAM;;AAI3E,QAAO;;AAGT,SAAS,WACP,OACA,WACA,UACA,MACO;AACP,KAAI,CAAC,2CAA2C,UAAU,CACxD,QAAO,oBAAoB,OAAO,WAAU,gBAC1C,YAAY,aAAa,WAAW,UAAU,KAAK,CACpD;AAGH,QAAO,qBAAqB,OAAO,WAAW,UAAU,KAAK;;AAG/D,SAAS,qBACP,OACA,WACA,UACA,MACO;CACP,IAAI,MAAM;AACV,QAAO,UAAU,SAAS,UAAU,sCAAsC;AAE1E,SAAQ,UAAU,MAAlB;EACE,KAAK,OAAO;GACV,MAAM,CAAC,aAAa,kBAAkB,kBACpC,UAAU,YACV,2CACD;AACD,OAAI,eAAe,SAAS,EAC1B,OAAM,oBAAoB,OAAO,WAAU,gBACzC,SACE,aACA;IACE,MAAM;IACN,YAAY;IACb,EACD,UACA,KACD,CACF;AAEH,UAAO,YAAY,SAAS,GAAG,mCAAmC;AAClE,QAAK,MAAM,QAAQ,YACjB,OAAM,qBAAqB,KAAK,MAAM,UAAU,KAAK;AAEvD;;EAEF,KAAK,MAAM;GACT,MAAM,CAAC,aAAa,kBAAkB,kBACpC,UAAU,YACV,2CACD;AACD,UAAO,YAAY,SAAS,GAAG,mCAAmC;GAElE,MAAM,MAAM,IAAI,YAAY,IAAI;AAChC,YAAS,QAAQ,KAAK,IAAI;AAC1B,SAAM,SAAS,cAAc,KAAK,GAAG,KAAK,MAAM;GAEhD,MAAM,WAAoB,EAAE;AAC5B,OAAI,eAAe,SAAS,EAC1B,UAAS,KACP,oBAAoB,KAAK,WAAU,gBACjC,QACE,aACA;IACE,MAAM;IACN,YAAY;IACb,EACD,UACA,KACD,CACF,CACF;AAGH,QAAK,MAAM,QAAQ,YACjB,UAAS,KAAK,qBAAqB,KAAK,MAAM,UAAU,KAAK,CAAC;GAGhE,MAAM,MAAM,IAAI,WAAW,KAAK,SAAS;AACzC,QAAK,MAAM,UAAU,SACnB,UAAS,QAAQ,QAAQ,IAAI;AAE/B,SAAM,SAAS,cAAc,KAAK,GAAG,KAAK,MAAM;AAEhD;;EAEF,KAAK,sBAAsB;GACzB,MAAM,KAAK,UAAU;GACrB,MAAM,QAAQ,sBACZ,GAAG,UACH,UACA,IACA,GAAG,KAAK,GAAG,GAAG,SAAS,SACvB,GAAG,YAAY,YACf,MACD;GACD,MAAM,cAAc,IAAI,YAAY;IAClC,QAAQ;IACR;IACA,WAAW,GAAG,YAAY;IAC1B,UAAU,GAAG,YAAY;IACzB,kBAAkB,KAChB,GAAG,SAAS,OACZ,8BACD;IACD,QAAQ,GAAG,UAAU;IACrB,QAAQ,GAAG,UAAU;IACtB,CAAC;AACF,YAAS,QAAQ,KAAK,YAAY;AAClC,YAAS,QAAQ,OAAO,YAAY;AACpC,SAAM,SAAS,cACb,aACA,GAAG,KAAK,gBAAgB,GAAG,SAAS,MAAM,GAC3C;AACD;;;AAIJ,QAAO;;AAGT,SAAS,YACP,OACA,WACA,UACA,MACa;AACb,SAAQ,UAAU,MAAlB;EACE,KAAK,MACH,QAAO,SAAS,OAAO,WAAW,UAAU,KAAK;EACnD,KAAK,KACH,QAAO,QAAQ,OAAO,WAAW,UAAU,KAAK;EAClD,KAAK,qBACH,QAAO,iCAAiC,OAAO,WAAW,UAAU,KAAK;EAC3E,KAAK,SACH,QAAO,qBAAqB,OAAO,UAAU,UAAU;;;AAI7D,SAAS,SACP,OACA,WACA,UACA,MACa;AACb,MAAK,MAAM,gBAAgB,UAAU,WACnC,SAAQ,YAAY,OAAO,cAAc,UAAU,KAAK;AAE1D,QAAO;;AAGT,SAAgB,QACd,OACA,WACA,UACA,MACa;CACb,MAAM,CAAC,oBAAoB,mBACzB,wBAAwB,UAAU;AAEpC,KAAI,mBAAmB,WAAW,GAAG;EACnC,MAAM,SAAS,IAAI,OACjB,OACA,gBAAgB;GACd,MAAM;GACN,YAAY;GACb,CAAC,CACH;AACD,WAAS,QAAQ,OAAO,OAAO;AAC/B,SAAO;;CAGT,MAAM,SAAS,IAAI,OAAO,MAAM;AAChC,UAAS,QAAQ,OAAO,OAAO;CAC/B,MAAM,WAAW,mBAAmB,KAAI,iBACtC,YAAY,QAAQ,cAAc,UAAU,KAAK,CAClD;AACD,KAAI,gBAAgB,SAAS,GAAG;EAC9B,MAAM,SAAS,IAAI,OACjB,QACA,gBAAgB;GACd,MAAM;GACN,YAAY;GACb,CAAC,CACH;AACD,WAAS,QAAQ,QAAQ,OAAO;AAChC,WAAS,KAAK,OAAO;;CAEvB,MAAM,MAAM,IAAI,MAAM,QAAQ,SAAS;AACvC,MAAK,MAAM,UAAU,SACnB,UAAS,QAAQ,QAAQ,IAAI;AAE/B,QAAO,SAAS,IAAI;AACpB,QAAO;;AAGT,SAAgB,wBAAwB,WAAwB;CAC9D,MAAM,cAGF,CAAC,EAAE,EAAE,EAAE,CAAC;AACZ,MAAK,MAAM,gBAAgB,UAAU,WACnC,KAAI,+BAA+B,aAAa,CAC9C,aAAY,GAAG,KAAK,aAAa;KAEjC,aAAY,GAAG,KAAK,aAAa;AAGrC,QAAO;;AAGT,SAAgB,+BACd,WACkC;AAClC,KAAI,UAAU,SAAS,qBACrB,QAAO;AAET,KAAI,UAAU,SAAS,SACrB,QAAO;AAET,QAAO,UAAU,WAAW,MAAM,+BAA+B;;AAGnE,SAAS,qBACP,OACA,UACA,WACa;CACb,MAAM,SAAS,IAAI,OAAO,OAAO,gBAAgB,UAAU,CAAC;AAC5D,UAAS,oBACP,QACA,GAAG,aAAa,UAAU,KAAK,CAAC,GAAG,UAAU,GAAG,GAAG,aAAa,UAAU,MAAM,GACjF;AACD,UAAS,QAAQ,OAAO,OAAO;AAC/B,QAAO;;AAGT,SAAS,aAAa,MAAqB;AACzC,SAAQ,KAAK,MAAb;EACE,KAAK,SACH,QAAO,KAAK;EACd,KAAK,UACH,QAAO,KAAK;EACd,KAAK,SACH,QAAO,KAAK;;;AAIlB,SAAS,wBACP,IACA,UACA,SACA,KACA,MACA,eACA;AAGA,KAAI,GAAG,SAAS,UAAU,KAAK,cAC7B,QAAO;AAGT,QAAO,GAAG,SAAS,OAAO,8BAA8B;CACxD,MAAM,QAAQ,sBACZ,GAAG,UACH,UACA,SACA,GAAG,KAAK,GAAG,GAAG,SAAS,SACvB,GAAG,YAAY,YACf,cACD;CAED,MAAM,WAAW,GAAG,KAAK,QAAQ,GAAG,SAAS,MAAM;CACnD,MAAM,OAAO,IAAI,KAAK;EACpB,QAAQ;EACR;EACA,WAAW,GAAG,YAAY;EAC1B,UAAU,GAAG,YAAY;EACzB,kBAAkB,GAAG,SAAS;EAC9B,QAAQ,GAAG,UAAU;EACrB,QAAQ,GAAG,UAAU;EACtB,CAAC;AACF,UAAS,QAAQ,KAAK,KAAK;AAC3B,UAAS,QAAQ,OAAO,KAAK;AAC7B,QAAO,SAAS,cAAc,MAAM,SAAS;;AAG/C,SAAS,iCACP,OACA,WACA,UACA,MACa;AACb,QACE,UAAU,OAAO,YAAY,UAAU,OAAO,cAC9C,yCACD;AACD,KAAI,UAAU,QAAQ,SAAS,UAAU,GAAG;AAC1C,MAAI,UAAU,OAAO,UAAU;GAC7B,MAAM,SAAS,IAAI,OAAO,aAAa,MAAM;AAC7C,YAAS,QAAQ,OAAO,OAAO;AAC/B,UAAO;;EAET,MAAM,SAAS,IAAI,OAAO,aAAa,KAAK;AAC5C,WAAS,QAAQ,OAAO,OAAO;AAC/B,SAAO;;CAET,MAAM,aAAa,GAAG,KAAK,UAAU,UAAU,QAAQ,SAAS,MAAM;CACtE,MAAM,SAAS,IAAI,OACjB,OACA,KAAK,UAAU,QAAQ,SAAS,MAAM,EACtC,UAAU,QAAQ,YAAY,aAC9B,UAAU,GACX;AACD,UAAS,QAAQ,OAAO,OAAO;AAC/B,QAAO,SAAS,oBAAoB,QAAQ,WAAW;;AAGzD,SAAS,wCACP,WACA;CACA,MAAM,OAAsC,EAAE;CAC9C,MAAM,UAAU,cAAyB;AACvC,MAAI,UAAU,SAAS,sBAAsB;AAC3C,QAAK,KAAK,UAAU;AACpB;;AAEF,MAAI,UAAU,SAAS,SAAS,UAAU,SAAS,MAAM;AACvD,QAAK,MAAM,KAAK,UAAU,WACxB,QAAO,EAAE;AAEX;;;AAGJ,KAAI,UACF,QAAO,UAAU;AAEnB,QAAO;;AAwBT,SAAS,2CAA2C,KAAe;AACjE,KAAI,CAAC,IAAI,MACP,QAAO;CAET,MAAM,EAAC,UAAS;AAChB,KAAI,MAAM,SAAS,SAAS,MAAM,SAAS,KACzC,QAAO;CAGT,IAAI,QAAQ;CACZ,MAAM,8BAA8B,UAAuC;EACzE,GAAG;EACH,SAAS;GACP,GAAG,KAAK;GACR,UAAU;IACR,GAAG,KAAK,QAAQ;IAChB,QAAQ,KAAK,QAAQ,SAAS,SAAS,MAAM,MAAM;IACpD;GACF;EACF;CAED,MAAM,YAAY,SAA+B;AAC/C,MAAI,KAAK,SAAS,SAChB,QAAO;WACE,KAAK,SAAS,qBACvB,QAAO,2BAA2B,KAAK;EAEzC,MAAM,aAAa,EAAE;AACrB,OAAK,MAAM,KAAK,KAAK,WACnB,YAAW,KAAK,SAAS,EAAE,CAAC;AAE9B,SAAO;GACL,MAAM,KAAK;GACX;GACD;;AAOH,QAJe;EACb,GAAG;EACH,OAAO,SAAS,MAAM;EACvB;;AAIH,SAAgB,2CACd,MACS;AACT,KAAI,KAAK,SAAS,qBAChB,QAAO,CAAC,CAAC,KAAK;AAEhB,KAAI,KAAK,SAAS,SAAS,KAAK,SAAS,KACvC,QAAO,KAAK,WAAW,MAAK,MAC1B,2CAA2C,EAAE,CAC9C;AAGH,QAAO;;AAGT,SAAgB,kBACd,YACA,WACA;CACA,MAAM,UAAuB,EAAE;CAC/B,MAAM,aAA0B,EAAE;AAClC,MAAK,MAAM,KAAK,WACd,KAAI,UAAU,EAAE,CACd,SAAQ,KAAK,EAAE;KAEf,YAAW,KAAK,EAAE;AAGtB,QAAO,CAAC,SAAS,WAAW"}
|
|
1
|
+
{"version":3,"file":"builder.js","names":[],"sources":["../../../../../zql/src/builder/builder.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert, unreachable} from '../../../shared/src/asserts.ts';\nimport type {JSONValue} from '../../../shared/src/json.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport type {\n AST,\n ColumnReference,\n CompoundKey,\n Condition,\n Conjunction,\n CorrelatedSubquery,\n CorrelatedSubqueryCondition,\n Disjunction,\n LiteralValue,\n Ordering,\n Parameter,\n SimpleCondition,\n ValuePosition,\n} from '../../../zero-protocol/src/ast.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts';\nimport {Exists} from '../ivm/exists.ts';\nimport {FanIn} from '../ivm/fan-in.ts';\nimport {FanOut} from '../ivm/fan-out.ts';\nimport {\n buildFilterPipeline,\n type FilterInput,\n} from '../ivm/filter-operators.ts';\nimport {Filter} from '../ivm/filter.ts';\nimport {FlippedJoin} from '../ivm/flipped-join.ts';\nimport {Join} from '../ivm/join.ts';\nimport type {Input, InputBase, Storage} from '../ivm/operator.ts';\nimport {Skip} from '../ivm/skip.ts';\nimport type {Source, SourceInput} from '../ivm/source.ts';\nimport {Take} from '../ivm/take.ts';\nimport {UnionFanIn} from '../ivm/union-fan-in.ts';\nimport {UnionFanOut} from '../ivm/union-fan-out.ts';\nimport {planQuery} from '../planner/planner-builder.ts';\nimport type {ConnectionCostModel} from '../planner/planner-connection.ts';\nimport type {PlanDebugger} from '../planner/planner-debug.ts';\nimport {completeOrdering} from '../query/complete-ordering.ts';\nimport type {DebugDelegate} from './debug-delegate.ts';\nimport {createPredicate, type NoSubqueryCondition} from './filter.ts';\n\nexport type StaticQueryParameters = {\n authData: Record<string, JSONValue>;\n preMutationRow?: Row | undefined;\n};\n\n/**\n * Interface required of caller to buildPipeline. Connects to constructed\n * pipeline to delegate environment to provide sources and storage.\n */\nexport interface BuilderDelegate {\n readonly applyFiltersAnyway?: boolean | undefined;\n debug?: DebugDelegate | undefined;\n\n /**\n * When true, allows NOT EXISTS conditions in queries.\n * Defaults to false.\n *\n * We only set this to true on the server.\n * The client-side query engine cannot support NOT EXISTS because:\n * 1. Zero only syncs a subset of data to the client\n * 2. On the client, we can't distinguish between a row not existing vs.\n * a row not being synced to the client\n * 3. NOT EXISTS requires complete knowledge of what doesn't exist\n */\n readonly enableNotExists?: boolean | undefined;\n\n /**\n * Called once for each source needed by the AST.\n * Might be called multiple times with same tableName. It is OK to return\n * same storage instance in that case.\n */\n getSource(tableName: string): Source | undefined;\n\n /**\n * Called once for each operator that requires storage. Should return a new\n * unique storage object for each call.\n */\n createStorage(name: string): Storage;\n\n decorateInput(input: Input, name: string): Input;\n\n addEdge(source: InputBase, dest: InputBase): void;\n\n decorateFilterInput(input: FilterInput, name: string): FilterInput;\n\n decorateSourceInput(input: SourceInput, queryID: string): Input;\n\n /**\n * The AST is mapped on-the-wire between client and server names.\n *\n * There is no \"wire\" for zqlite tests so this function is provided\n * to allow tests to remap the AST.\n */\n mapAst?: ((ast: AST) => AST) | undefined;\n}\n\n/**\n * Builds a pipeline from an AST. Caller must provide a delegate to create source\n * and storage interfaces as necessary.\n *\n * Usage:\n *\n * ```ts\n * class MySink implements Output {\n * readonly #input: Input;\n *\n * constructor(input: Input) {\n * this.#input = input;\n * input.setOutput(this);\n * }\n *\n * push(change: Change, _: Operator) {\n * console.log(change);\n * }\n * }\n *\n * const input = buildPipeline(ast, myDelegate, hash(ast));\n * const sink = new MySink(input);\n * ```\n */\nexport function buildPipeline(\n ast: AST,\n delegate: BuilderDelegate,\n queryID: string,\n costModel?: ConnectionCostModel,\n lc?: LogContext,\n planDebugger?: PlanDebugger,\n): Input {\n ast = delegate.mapAst ? delegate.mapAst(ast) : ast;\n ast = completeOrdering(\n ast,\n tableName => must(delegate.getSource(tableName)).tableSchema.primaryKey,\n );\n\n if (costModel) {\n ast = planQuery(ast, costModel, planDebugger, lc);\n }\n return buildPipelineInternal(ast, delegate, queryID, '');\n}\n\nexport function bindStaticParameters(\n ast: AST,\n staticQueryParameters: StaticQueryParameters | undefined,\n) {\n const visit = (node: AST): AST => ({\n ...node,\n where: node.where ? bindCondition(node.where) : undefined,\n related: node.related?.map(sq => ({\n ...sq,\n subquery: visit(sq.subquery),\n })),\n });\n\n function bindCondition(condition: Condition): Condition {\n if (condition.type === 'simple') {\n return {\n ...condition,\n left: bindValue(condition.left),\n right: bindValue(condition.right) as Exclude<\n ValuePosition,\n ColumnReference\n >,\n };\n }\n if (condition.type === 'correlatedSubquery') {\n return {\n ...condition,\n related: {\n ...condition.related,\n subquery: visit(condition.related.subquery),\n },\n };\n }\n\n return {\n ...condition,\n conditions: condition.conditions.map(bindCondition),\n };\n }\n\n const bindValue = (value: ValuePosition): ValuePosition => {\n if (isParameter(value)) {\n const anchor = must(\n staticQueryParameters,\n 'Static query params do not exist',\n )[value.anchor];\n const resolvedValue = resolveField(anchor, value.field);\n return {\n type: 'literal',\n value: resolvedValue as LiteralValue,\n };\n }\n return value;\n };\n\n return visit(ast);\n}\n\nfunction resolveField(\n anchor: Record<string, JSONValue> | Row | undefined,\n field: string | string[],\n): unknown {\n if (anchor === undefined) {\n return null;\n }\n\n if (Array.isArray(field)) {\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n return field.reduce((acc, f) => (acc as any)?.[f], anchor) ?? null;\n }\n\n return anchor[field] ?? null;\n}\n\nfunction isParameter(value: ValuePosition): value is Parameter {\n return value.type === 'static';\n}\n\nconst EXISTS_LIMIT = 3;\nconst PERMISSIONS_EXISTS_LIMIT = 1;\n\n/**\n * Checks if a condition tree contains any NOT EXISTS operations.\n * Recursively checks AND/OR branches but does not recurse into nested subqueries\n * (those are checked when buildPipelineInternal processes them).\n */\nexport function assertNoNotExists(condition: Condition): void {\n switch (condition.type) {\n case 'simple':\n return;\n\n case 'correlatedSubquery':\n if (condition.op === 'NOT EXISTS') {\n throw new Error(\n 'not(exists()) is not supported on the client - see https://bugs.rocicorp.dev/issue/3438',\n );\n }\n return;\n\n case 'and':\n case 'or':\n for (const c of condition.conditions) {\n assertNoNotExists(c);\n }\n return;\n default:\n unreachable(condition);\n }\n}\n\nfunction buildPipelineInternal(\n ast: AST,\n delegate: BuilderDelegate,\n queryID: string,\n name: string,\n partitionKey?: CompoundKey,\n): Input {\n const source = delegate.getSource(ast.table);\n if (!source) {\n throw new Error(`Source not found: ${ast.table}`);\n }\n\n ast = uniquifyCorrelatedSubqueryConditionAliases(ast);\n\n if (!delegate.enableNotExists && ast.where) {\n assertNoNotExists(ast.where);\n }\n\n const csqConditions = gatherCorrelatedSubqueryQueryConditions(ast.where);\n const splitEditKeys: Set<string> = partitionKey\n ? new Set(partitionKey)\n : new Set();\n const aliases = new Set<string>();\n for (const csq of csqConditions) {\n aliases.add(csq.related.subquery.alias || '');\n for (const key of csq.related.correlation.parentField) {\n splitEditKeys.add(key);\n }\n }\n if (ast.related) {\n for (const csq of ast.related) {\n for (const key of csq.correlation.parentField) {\n splitEditKeys.add(key);\n }\n }\n }\n const conn = source.connect(\n must(ast.orderBy),\n ast.where,\n splitEditKeys,\n delegate.debug,\n );\n\n let end: Input = delegate.decorateSourceInput(conn, queryID);\n end = delegate.decorateInput(end, `${name}:source(${ast.table})`);\n const {fullyAppliedFilters} = conn;\n\n if (ast.start) {\n const skip = new Skip(end, ast.start);\n delegate.addEdge(end, skip);\n end = delegate.decorateInput(skip, `${name}:skip)`);\n }\n\n for (const csqCondition of csqConditions) {\n // flipped EXISTS are handled in applyWhere\n if (!csqCondition.flip) {\n end = applyCorrelatedSubQuery(\n {\n ...csqCondition.related,\n subquery: {\n ...csqCondition.related.subquery,\n limit:\n csqCondition.related.system === 'permissions'\n ? PERMISSIONS_EXISTS_LIMIT\n : EXISTS_LIMIT,\n },\n },\n delegate,\n queryID,\n end,\n name,\n true,\n );\n }\n }\n\n if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) {\n end = applyWhere(end, ast.where, delegate, name);\n }\n\n if (ast.limit !== undefined) {\n const takeName = `${name}:take`;\n const take = new Take(\n end,\n delegate.createStorage(takeName),\n ast.limit,\n partitionKey,\n );\n delegate.addEdge(end, take);\n end = delegate.decorateInput(take, takeName);\n }\n\n if (ast.related) {\n // Dedupe by alias - last one wins (LWW), like limit(5).limit(10)\n const byAlias = new Map<string, CorrelatedSubquery>();\n for (const csq of ast.related) {\n byAlias.set(csq.subquery.alias ?? '', csq);\n }\n for (const csq of byAlias.values()) {\n end = applyCorrelatedSubQuery(csq, delegate, queryID, end, name, false);\n }\n }\n\n return end;\n}\n\nfunction applyWhere(\n input: Input,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): Input {\n if (!conditionIncludesFlippedSubqueryAtAnyLevel(condition)) {\n return buildFilterPipeline(input, delegate, filterInput =>\n applyFilter(filterInput, condition, delegate, name),\n );\n }\n\n return applyFilterWithFlips(input, condition, delegate, name);\n}\n\nfunction applyFilterWithFlips(\n input: Input,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): Input {\n let end = input;\n assert(condition.type !== 'simple', 'Simple conditions cannot have flips');\n\n switch (condition.type) {\n case 'and': {\n const [withFlipped, withoutFlipped] = partitionBranches(\n condition.conditions,\n conditionIncludesFlippedSubqueryAtAnyLevel,\n );\n if (withoutFlipped.length > 0) {\n end = buildFilterPipeline(input, delegate, filterInput =>\n applyAnd(\n filterInput,\n {\n type: 'and',\n conditions: withoutFlipped,\n },\n delegate,\n name,\n ),\n );\n }\n assert(withFlipped.length > 0, 'Impossible to have no flips here');\n for (const cond of withFlipped) {\n end = applyFilterWithFlips(end, cond, delegate, name);\n }\n break;\n }\n case 'or': {\n const [withFlipped, withoutFlipped] = partitionBranches(\n condition.conditions,\n conditionIncludesFlippedSubqueryAtAnyLevel,\n );\n assert(withFlipped.length > 0, 'Impossible to have no flips here');\n\n const ufo = new UnionFanOut(end);\n delegate.addEdge(end, ufo);\n end = delegate.decorateInput(ufo, `${name}:ufo`);\n\n const branches: Input[] = [];\n if (withoutFlipped.length > 0) {\n branches.push(\n buildFilterPipeline(end, delegate, filterInput =>\n applyOr(\n filterInput,\n {\n type: 'or',\n conditions: withoutFlipped,\n },\n delegate,\n name,\n ),\n ),\n );\n }\n\n for (const cond of withFlipped) {\n branches.push(applyFilterWithFlips(end, cond, delegate, name));\n }\n\n const ufi = new UnionFanIn(ufo, branches);\n for (const branch of branches) {\n delegate.addEdge(branch, ufi);\n }\n end = delegate.decorateInput(ufi, `${name}:ufi`);\n\n break;\n }\n case 'correlatedSubquery': {\n const sq = condition.related;\n const child = buildPipelineInternal(\n sq.subquery,\n delegate,\n '',\n `${name}.${sq.subquery.alias}`,\n sq.correlation.childField,\n );\n const flippedJoin = new FlippedJoin({\n parent: end,\n child,\n parentKey: sq.correlation.parentField,\n childKey: sq.correlation.childField,\n relationshipName: must(\n sq.subquery.alias,\n 'Subquery must have an alias',\n ),\n hidden: sq.hidden ?? false,\n system: sq.system ?? 'client',\n });\n delegate.addEdge(end, flippedJoin);\n delegate.addEdge(child, flippedJoin);\n end = delegate.decorateInput(\n flippedJoin,\n `${name}:flipped-join(${sq.subquery.alias})`,\n );\n break;\n }\n }\n\n return end;\n}\n\nfunction applyFilter(\n input: FilterInput,\n condition: Condition,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n switch (condition.type) {\n case 'and':\n return applyAnd(input, condition, delegate, name);\n case 'or':\n return applyOr(input, condition, delegate, name);\n case 'correlatedSubquery':\n return applyCorrelatedSubqueryCondition(input, condition, delegate, name);\n case 'simple':\n return applySimpleCondition(input, delegate, condition);\n }\n}\n\nfunction applyAnd(\n input: FilterInput,\n condition: Conjunction,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n for (const subCondition of condition.conditions) {\n input = applyFilter(input, subCondition, delegate, name);\n }\n return input;\n}\n\nexport function applyOr(\n input: FilterInput,\n condition: Disjunction,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n const [subqueryConditions, otherConditions] =\n groupSubqueryConditions(condition);\n // if there are no subquery conditions, no fan-in / fan-out is needed\n if (subqueryConditions.length === 0) {\n const filter = new Filter(\n input,\n createPredicate({\n type: 'or',\n conditions: otherConditions,\n }),\n );\n delegate.addEdge(input, filter);\n return filter;\n }\n\n const fanOut = new FanOut(input);\n delegate.addEdge(input, fanOut);\n const branches = subqueryConditions.map(subCondition =>\n applyFilter(fanOut, subCondition, delegate, name),\n );\n if (otherConditions.length > 0) {\n const filter = new Filter(\n fanOut,\n createPredicate({\n type: 'or',\n conditions: otherConditions,\n }),\n );\n delegate.addEdge(fanOut, filter);\n branches.push(filter);\n }\n const ret = new FanIn(fanOut, branches);\n for (const branch of branches) {\n delegate.addEdge(branch, ret);\n }\n fanOut.setFanIn(ret);\n return ret;\n}\n\nexport function groupSubqueryConditions(condition: Disjunction) {\n const partitioned: [\n subqueryConditions: Condition[],\n otherConditions: NoSubqueryCondition[],\n ] = [[], []];\n for (const subCondition of condition.conditions) {\n if (isNotAndDoesNotContainSubquery(subCondition)) {\n partitioned[1].push(subCondition);\n } else {\n partitioned[0].push(subCondition);\n }\n }\n return partitioned;\n}\n\nexport function isNotAndDoesNotContainSubquery(\n condition: Condition,\n): condition is NoSubqueryCondition {\n if (condition.type === 'correlatedSubquery') {\n return false;\n }\n if (condition.type === 'simple') {\n return true;\n }\n return condition.conditions.every(isNotAndDoesNotContainSubquery);\n}\n\nfunction applySimpleCondition(\n input: FilterInput,\n delegate: BuilderDelegate,\n condition: SimpleCondition,\n): FilterInput {\n const filter = new Filter(input, createPredicate(condition));\n delegate.decorateFilterInput(\n filter,\n `${valuePosName(condition.left)}:${condition.op}:${valuePosName(condition.right)}`,\n );\n delegate.addEdge(input, filter);\n return filter;\n}\n\nfunction valuePosName(left: ValuePosition) {\n switch (left.type) {\n case 'static':\n return left.field;\n case 'literal':\n return left.value;\n case 'column':\n return left.name;\n }\n}\n\nfunction applyCorrelatedSubQuery(\n sq: CorrelatedSubquery,\n delegate: BuilderDelegate,\n queryID: string,\n end: Input,\n name: string,\n fromCondition: boolean,\n) {\n // TODO: we only omit the join if the CSQ if from a condition since\n // we want to create an empty array for `related` fields that are `limit(0)`\n if (sq.subquery.limit === 0 && fromCondition) {\n return end;\n }\n\n assert(sq.subquery.alias, 'Subquery must have an alias');\n const child = buildPipelineInternal(\n sq.subquery,\n delegate,\n queryID,\n `${name}.${sq.subquery.alias}`,\n sq.correlation.childField,\n );\n\n const joinName = `${name}:join(${sq.subquery.alias})`;\n const join = new Join({\n parent: end,\n child,\n parentKey: sq.correlation.parentField,\n childKey: sq.correlation.childField,\n relationshipName: sq.subquery.alias,\n hidden: sq.hidden ?? false,\n system: sq.system ?? 'client',\n });\n delegate.addEdge(end, join);\n delegate.addEdge(child, join);\n return delegate.decorateInput(join, joinName);\n}\n\nfunction applyCorrelatedSubqueryCondition(\n input: FilterInput,\n condition: CorrelatedSubqueryCondition,\n delegate: BuilderDelegate,\n name: string,\n): FilterInput {\n assert(\n condition.op === 'EXISTS' || condition.op === 'NOT EXISTS',\n 'Expected EXISTS or NOT EXISTS operator',\n );\n if (condition.related.subquery.limit === 0) {\n if (condition.op === 'EXISTS') {\n const filter = new Filter(input, () => false);\n delegate.addEdge(input, filter);\n return filter;\n }\n const filter = new Filter(input, () => true);\n delegate.addEdge(input, filter);\n return filter;\n }\n const existsName = `${name}:exists(${condition.related.subquery.alias})`;\n const exists = new Exists(\n input,\n must(condition.related.subquery.alias),\n condition.related.correlation.parentField,\n condition.op,\n );\n delegate.addEdge(input, exists);\n return delegate.decorateFilterInput(exists, existsName);\n}\n\nfunction gatherCorrelatedSubqueryQueryConditions(\n condition: Condition | undefined,\n) {\n const csqs: CorrelatedSubqueryCondition[] = [];\n const gather = (condition: Condition) => {\n if (condition.type === 'correlatedSubquery') {\n csqs.push(condition);\n return;\n }\n if (condition.type === 'and' || condition.type === 'or') {\n for (const c of condition.conditions) {\n gather(c);\n }\n return;\n }\n };\n if (condition) {\n gather(condition);\n }\n return csqs;\n}\n\nexport function assertOrderingIncludesPK(\n ordering: Ordering,\n pk: PrimaryKey,\n): void {\n // oxlint-disable-next-line unicorn/prefer-set-has -- Array is more appropriate here for small collections\n const orderingFields = ordering.map(([field]) => field);\n const missingFields = pk.filter(pkField => !orderingFields.includes(pkField));\n\n if (missingFields.length > 0) {\n throw new Error(\n `Ordering must include all primary key fields. Missing: ${missingFields.join(\n ', ',\n )}. ZQL automatically appends primary key fields to the ordering if they are missing \n so a common cause of this error is a casing mismatch between Postgres and ZQL.\n E.g., \"userid\" vs \"userID\".\n You may want to add double-quotes around your Postgres column names to prevent Postgres from lower-casing them:\n https://www.postgresql.org/docs/current/sql-syntax-lexical.htm`,\n );\n }\n}\n\nfunction uniquifyCorrelatedSubqueryConditionAliases(ast: AST): AST {\n if (!ast.where) {\n return ast;\n }\n const {where} = ast;\n if (where.type !== 'and' && where.type !== 'or') {\n return ast;\n }\n\n let count = 0;\n const uniquifyCorrelatedSubquery = (csqc: CorrelatedSubqueryCondition) => ({\n ...csqc,\n related: {\n ...csqc.related,\n subquery: {\n ...csqc.related.subquery,\n alias: (csqc.related.subquery.alias ?? '') + '_' + count++,\n },\n },\n });\n\n const uniquify = (cond: Condition): Condition => {\n if (cond.type === 'simple') {\n return cond;\n } else if (cond.type === 'correlatedSubquery') {\n return uniquifyCorrelatedSubquery(cond);\n }\n const conditions = [];\n for (const c of cond.conditions) {\n conditions.push(uniquify(c));\n }\n return {\n type: cond.type,\n conditions,\n };\n };\n\n const result = {\n ...ast,\n where: uniquify(where),\n };\n return result;\n}\n\nexport function conditionIncludesFlippedSubqueryAtAnyLevel(\n cond: Condition,\n): boolean {\n if (cond.type === 'correlatedSubquery') {\n return !!cond.flip;\n }\n if (cond.type === 'and' || cond.type === 'or') {\n return cond.conditions.some(c =>\n conditionIncludesFlippedSubqueryAtAnyLevel(c),\n );\n }\n // simple conditions don't have flips\n return false;\n}\n\nexport function partitionBranches(\n conditions: readonly Condition[],\n predicate: (c: Condition) => boolean,\n) {\n const matched: Condition[] = [];\n const notMatched: Condition[] = [];\n for (const c of conditions) {\n if (predicate(c)) {\n matched.push(c);\n } else {\n notMatched.push(c);\n }\n }\n return [matched, notMatched] as const;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4HA,SAAgB,cACd,KACA,UACA,SACA,WACA,IACA,cACO;AACP,OAAM,SAAS,SAAS,SAAS,OAAO,IAAI,GAAG;AAC/C,OAAM,iBACJ,MACA,cAAa,KAAK,SAAS,UAAU,UAAU,CAAC,CAAC,YAAY,WAC9D;AAED,KAAI,UACF,OAAM,UAAU,KAAK,WAAW,cAAc,GAAG;AAEnD,QAAO,sBAAsB,KAAK,UAAU,SAAS,GAAG;;AAG1D,SAAgB,qBACd,KACA,uBACA;CACA,MAAM,SAAS,UAAoB;EACjC,GAAG;EACH,OAAO,KAAK,QAAQ,cAAc,KAAK,MAAM,GAAG,KAAA;EAChD,SAAS,KAAK,SAAS,KAAI,QAAO;GAChC,GAAG;GACH,UAAU,MAAM,GAAG,SAAS;GAC7B,EAAE;EACJ;CAED,SAAS,cAAc,WAAiC;AACtD,MAAI,UAAU,SAAS,SACrB,QAAO;GACL,GAAG;GACH,MAAM,UAAU,UAAU,KAAK;GAC/B,OAAO,UAAU,UAAU,MAAM;GAIlC;AAEH,MAAI,UAAU,SAAS,qBACrB,QAAO;GACL,GAAG;GACH,SAAS;IACP,GAAG,UAAU;IACb,UAAU,MAAM,UAAU,QAAQ,SAAS;IAC5C;GACF;AAGH,SAAO;GACL,GAAG;GACH,YAAY,UAAU,WAAW,IAAI,cAAc;GACpD;;CAGH,MAAM,aAAa,UAAwC;AACzD,MAAI,YAAY,MAAM,EAAE;GACtB,MAAM,SAAS,KACb,uBACA,mCACD,CAAC,MAAM;AAER,UAAO;IACL,MAAM;IACN,OAHoB,aAAa,QAAQ,MAAM,MAAM;IAItD;;AAEH,SAAO;;AAGT,QAAO,MAAM,IAAI;;AAGnB,SAAS,aACP,QACA,OACS;AACT,KAAI,WAAW,KAAA,EACb,QAAO;AAGT,KAAI,MAAM,QAAQ,MAAM,CAEtB,QAAO,MAAM,QAAQ,KAAK,MAAO,MAAc,IAAI,OAAO,IAAI;AAGhE,QAAO,OAAO,UAAU;;AAG1B,SAAS,YAAY,OAA0C;AAC7D,QAAO,MAAM,SAAS;;AAGxB,IAAM,eAAe;AACrB,IAAM,2BAA2B;;;;;;AAOjC,SAAgB,kBAAkB,WAA4B;AAC5D,SAAQ,UAAU,MAAlB;EACE,KAAK,SACH;EAEF,KAAK;AACH,OAAI,UAAU,OAAO,aACnB,OAAM,IAAI,MACR,0FACD;AAEH;EAEF,KAAK;EACL,KAAK;AACH,QAAK,MAAM,KAAK,UAAU,WACxB,mBAAkB,EAAE;AAEtB;EACF,QACE,aAAY,UAAU;;;AAI5B,SAAS,sBACP,KACA,UACA,SACA,MACA,cACO;CACP,MAAM,SAAS,SAAS,UAAU,IAAI,MAAM;AAC5C,KAAI,CAAC,OACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,QAAQ;AAGnD,OAAM,2CAA2C,IAAI;AAErD,KAAI,CAAC,SAAS,mBAAmB,IAAI,MACnC,mBAAkB,IAAI,MAAM;CAG9B,MAAM,gBAAgB,wCAAwC,IAAI,MAAM;CACxE,MAAM,gBAA6B,eAC/B,IAAI,IAAI,aAAa,mBACrB,IAAI,KAAK;CACb,MAAM,0BAAU,IAAI,KAAa;AACjC,MAAK,MAAM,OAAO,eAAe;AAC/B,UAAQ,IAAI,IAAI,QAAQ,SAAS,SAAS,GAAG;AAC7C,OAAK,MAAM,OAAO,IAAI,QAAQ,YAAY,YACxC,eAAc,IAAI,IAAI;;AAG1B,KAAI,IAAI,QACN,MAAK,MAAM,OAAO,IAAI,QACpB,MAAK,MAAM,OAAO,IAAI,YAAY,YAChC,eAAc,IAAI,IAAI;CAI5B,MAAM,OAAO,OAAO,QAClB,KAAK,IAAI,QAAQ,EACjB,IAAI,OACJ,eACA,SAAS,MACV;CAED,IAAI,MAAa,SAAS,oBAAoB,MAAM,QAAQ;AAC5D,OAAM,SAAS,cAAc,KAAK,GAAG,KAAK,UAAU,IAAI,MAAM,GAAG;CACjE,MAAM,EAAC,wBAAuB;AAE9B,KAAI,IAAI,OAAO;EACb,MAAM,OAAO,IAAI,KAAK,KAAK,IAAI,MAAM;AACrC,WAAS,QAAQ,KAAK,KAAK;AAC3B,QAAM,SAAS,cAAc,MAAM,GAAG,KAAK,QAAQ;;AAGrD,MAAK,MAAM,gBAAgB,cAEzB,KAAI,CAAC,aAAa,KAChB,OAAM,wBACJ;EACE,GAAG,aAAa;EAChB,UAAU;GACR,GAAG,aAAa,QAAQ;GACxB,OACE,aAAa,QAAQ,WAAW,gBAC5B,2BACA;GACP;EACF,EACD,UACA,SACA,KACA,MACA,KACD;AAIL,KAAI,IAAI,UAAU,CAAC,uBAAuB,SAAS,oBACjD,OAAM,WAAW,KAAK,IAAI,OAAO,UAAU,KAAK;AAGlD,KAAI,IAAI,UAAU,KAAA,GAAW;EAC3B,MAAM,WAAW,GAAG,KAAK;EACzB,MAAM,OAAO,IAAI,KACf,KACA,SAAS,cAAc,SAAS,EAChC,IAAI,OACJ,aACD;AACD,WAAS,QAAQ,KAAK,KAAK;AAC3B,QAAM,SAAS,cAAc,MAAM,SAAS;;AAG9C,KAAI,IAAI,SAAS;EAEf,MAAM,0BAAU,IAAI,KAAiC;AACrD,OAAK,MAAM,OAAO,IAAI,QACpB,SAAQ,IAAI,IAAI,SAAS,SAAS,IAAI,IAAI;AAE5C,OAAK,MAAM,OAAO,QAAQ,QAAQ,CAChC,OAAM,wBAAwB,KAAK,UAAU,SAAS,KAAK,MAAM,MAAM;;AAI3E,QAAO;;AAGT,SAAS,WACP,OACA,WACA,UACA,MACO;AACP,KAAI,CAAC,2CAA2C,UAAU,CACxD,QAAO,oBAAoB,OAAO,WAAU,gBAC1C,YAAY,aAAa,WAAW,UAAU,KAAK,CACpD;AAGH,QAAO,qBAAqB,OAAO,WAAW,UAAU,KAAK;;AAG/D,SAAS,qBACP,OACA,WACA,UACA,MACO;CACP,IAAI,MAAM;AACV,QAAO,UAAU,SAAS,UAAU,sCAAsC;AAE1E,SAAQ,UAAU,MAAlB;EACE,KAAK,OAAO;GACV,MAAM,CAAC,aAAa,kBAAkB,kBACpC,UAAU,YACV,2CACD;AACD,OAAI,eAAe,SAAS,EAC1B,OAAM,oBAAoB,OAAO,WAAU,gBACzC,SACE,aACA;IACE,MAAM;IACN,YAAY;IACb,EACD,UACA,KACD,CACF;AAEH,UAAO,YAAY,SAAS,GAAG,mCAAmC;AAClE,QAAK,MAAM,QAAQ,YACjB,OAAM,qBAAqB,KAAK,MAAM,UAAU,KAAK;AAEvD;;EAEF,KAAK,MAAM;GACT,MAAM,CAAC,aAAa,kBAAkB,kBACpC,UAAU,YACV,2CACD;AACD,UAAO,YAAY,SAAS,GAAG,mCAAmC;GAElE,MAAM,MAAM,IAAI,YAAY,IAAI;AAChC,YAAS,QAAQ,KAAK,IAAI;AAC1B,SAAM,SAAS,cAAc,KAAK,GAAG,KAAK,MAAM;GAEhD,MAAM,WAAoB,EAAE;AAC5B,OAAI,eAAe,SAAS,EAC1B,UAAS,KACP,oBAAoB,KAAK,WAAU,gBACjC,QACE,aACA;IACE,MAAM;IACN,YAAY;IACb,EACD,UACA,KACD,CACF,CACF;AAGH,QAAK,MAAM,QAAQ,YACjB,UAAS,KAAK,qBAAqB,KAAK,MAAM,UAAU,KAAK,CAAC;GAGhE,MAAM,MAAM,IAAI,WAAW,KAAK,SAAS;AACzC,QAAK,MAAM,UAAU,SACnB,UAAS,QAAQ,QAAQ,IAAI;AAE/B,SAAM,SAAS,cAAc,KAAK,GAAG,KAAK,MAAM;AAEhD;;EAEF,KAAK,sBAAsB;GACzB,MAAM,KAAK,UAAU;GACrB,MAAM,QAAQ,sBACZ,GAAG,UACH,UACA,IACA,GAAG,KAAK,GAAG,GAAG,SAAS,SACvB,GAAG,YAAY,WAChB;GACD,MAAM,cAAc,IAAI,YAAY;IAClC,QAAQ;IACR;IACA,WAAW,GAAG,YAAY;IAC1B,UAAU,GAAG,YAAY;IACzB,kBAAkB,KAChB,GAAG,SAAS,OACZ,8BACD;IACD,QAAQ,GAAG,UAAU;IACrB,QAAQ,GAAG,UAAU;IACtB,CAAC;AACF,YAAS,QAAQ,KAAK,YAAY;AAClC,YAAS,QAAQ,OAAO,YAAY;AACpC,SAAM,SAAS,cACb,aACA,GAAG,KAAK,gBAAgB,GAAG,SAAS,MAAM,GAC3C;AACD;;;AAIJ,QAAO;;AAGT,SAAS,YACP,OACA,WACA,UACA,MACa;AACb,SAAQ,UAAU,MAAlB;EACE,KAAK,MACH,QAAO,SAAS,OAAO,WAAW,UAAU,KAAK;EACnD,KAAK,KACH,QAAO,QAAQ,OAAO,WAAW,UAAU,KAAK;EAClD,KAAK,qBACH,QAAO,iCAAiC,OAAO,WAAW,UAAU,KAAK;EAC3E,KAAK,SACH,QAAO,qBAAqB,OAAO,UAAU,UAAU;;;AAI7D,SAAS,SACP,OACA,WACA,UACA,MACa;AACb,MAAK,MAAM,gBAAgB,UAAU,WACnC,SAAQ,YAAY,OAAO,cAAc,UAAU,KAAK;AAE1D,QAAO;;AAGT,SAAgB,QACd,OACA,WACA,UACA,MACa;CACb,MAAM,CAAC,oBAAoB,mBACzB,wBAAwB,UAAU;AAEpC,KAAI,mBAAmB,WAAW,GAAG;EACnC,MAAM,SAAS,IAAI,OACjB,OACA,gBAAgB;GACd,MAAM;GACN,YAAY;GACb,CAAC,CACH;AACD,WAAS,QAAQ,OAAO,OAAO;AAC/B,SAAO;;CAGT,MAAM,SAAS,IAAI,OAAO,MAAM;AAChC,UAAS,QAAQ,OAAO,OAAO;CAC/B,MAAM,WAAW,mBAAmB,KAAI,iBACtC,YAAY,QAAQ,cAAc,UAAU,KAAK,CAClD;AACD,KAAI,gBAAgB,SAAS,GAAG;EAC9B,MAAM,SAAS,IAAI,OACjB,QACA,gBAAgB;GACd,MAAM;GACN,YAAY;GACb,CAAC,CACH;AACD,WAAS,QAAQ,QAAQ,OAAO;AAChC,WAAS,KAAK,OAAO;;CAEvB,MAAM,MAAM,IAAI,MAAM,QAAQ,SAAS;AACvC,MAAK,MAAM,UAAU,SACnB,UAAS,QAAQ,QAAQ,IAAI;AAE/B,QAAO,SAAS,IAAI;AACpB,QAAO;;AAGT,SAAgB,wBAAwB,WAAwB;CAC9D,MAAM,cAGF,CAAC,EAAE,EAAE,EAAE,CAAC;AACZ,MAAK,MAAM,gBAAgB,UAAU,WACnC,KAAI,+BAA+B,aAAa,CAC9C,aAAY,GAAG,KAAK,aAAa;KAEjC,aAAY,GAAG,KAAK,aAAa;AAGrC,QAAO;;AAGT,SAAgB,+BACd,WACkC;AAClC,KAAI,UAAU,SAAS,qBACrB,QAAO;AAET,KAAI,UAAU,SAAS,SACrB,QAAO;AAET,QAAO,UAAU,WAAW,MAAM,+BAA+B;;AAGnE,SAAS,qBACP,OACA,UACA,WACa;CACb,MAAM,SAAS,IAAI,OAAO,OAAO,gBAAgB,UAAU,CAAC;AAC5D,UAAS,oBACP,QACA,GAAG,aAAa,UAAU,KAAK,CAAC,GAAG,UAAU,GAAG,GAAG,aAAa,UAAU,MAAM,GACjF;AACD,UAAS,QAAQ,OAAO,OAAO;AAC/B,QAAO;;AAGT,SAAS,aAAa,MAAqB;AACzC,SAAQ,KAAK,MAAb;EACE,KAAK,SACH,QAAO,KAAK;EACd,KAAK,UACH,QAAO,KAAK;EACd,KAAK,SACH,QAAO,KAAK;;;AAIlB,SAAS,wBACP,IACA,UACA,SACA,KACA,MACA,eACA;AAGA,KAAI,GAAG,SAAS,UAAU,KAAK,cAC7B,QAAO;AAGT,QAAO,GAAG,SAAS,OAAO,8BAA8B;CACxD,MAAM,QAAQ,sBACZ,GAAG,UACH,UACA,SACA,GAAG,KAAK,GAAG,GAAG,SAAS,SACvB,GAAG,YAAY,WAChB;CAED,MAAM,WAAW,GAAG,KAAK,QAAQ,GAAG,SAAS,MAAM;CACnD,MAAM,OAAO,IAAI,KAAK;EACpB,QAAQ;EACR;EACA,WAAW,GAAG,YAAY;EAC1B,UAAU,GAAG,YAAY;EACzB,kBAAkB,GAAG,SAAS;EAC9B,QAAQ,GAAG,UAAU;EACrB,QAAQ,GAAG,UAAU;EACtB,CAAC;AACF,UAAS,QAAQ,KAAK,KAAK;AAC3B,UAAS,QAAQ,OAAO,KAAK;AAC7B,QAAO,SAAS,cAAc,MAAM,SAAS;;AAG/C,SAAS,iCACP,OACA,WACA,UACA,MACa;AACb,QACE,UAAU,OAAO,YAAY,UAAU,OAAO,cAC9C,yCACD;AACD,KAAI,UAAU,QAAQ,SAAS,UAAU,GAAG;AAC1C,MAAI,UAAU,OAAO,UAAU;GAC7B,MAAM,SAAS,IAAI,OAAO,aAAa,MAAM;AAC7C,YAAS,QAAQ,OAAO,OAAO;AAC/B,UAAO;;EAET,MAAM,SAAS,IAAI,OAAO,aAAa,KAAK;AAC5C,WAAS,QAAQ,OAAO,OAAO;AAC/B,SAAO;;CAET,MAAM,aAAa,GAAG,KAAK,UAAU,UAAU,QAAQ,SAAS,MAAM;CACtE,MAAM,SAAS,IAAI,OACjB,OACA,KAAK,UAAU,QAAQ,SAAS,MAAM,EACtC,UAAU,QAAQ,YAAY,aAC9B,UAAU,GACX;AACD,UAAS,QAAQ,OAAO,OAAO;AAC/B,QAAO,SAAS,oBAAoB,QAAQ,WAAW;;AAGzD,SAAS,wCACP,WACA;CACA,MAAM,OAAsC,EAAE;CAC9C,MAAM,UAAU,cAAyB;AACvC,MAAI,UAAU,SAAS,sBAAsB;AAC3C,QAAK,KAAK,UAAU;AACpB;;AAEF,MAAI,UAAU,SAAS,SAAS,UAAU,SAAS,MAAM;AACvD,QAAK,MAAM,KAAK,UAAU,WACxB,QAAO,EAAE;AAEX;;;AAGJ,KAAI,UACF,QAAO,UAAU;AAEnB,QAAO;;AAwBT,SAAS,2CAA2C,KAAe;AACjE,KAAI,CAAC,IAAI,MACP,QAAO;CAET,MAAM,EAAC,UAAS;AAChB,KAAI,MAAM,SAAS,SAAS,MAAM,SAAS,KACzC,QAAO;CAGT,IAAI,QAAQ;CACZ,MAAM,8BAA8B,UAAuC;EACzE,GAAG;EACH,SAAS;GACP,GAAG,KAAK;GACR,UAAU;IACR,GAAG,KAAK,QAAQ;IAChB,QAAQ,KAAK,QAAQ,SAAS,SAAS,MAAM,MAAM;IACpD;GACF;EACF;CAED,MAAM,YAAY,SAA+B;AAC/C,MAAI,KAAK,SAAS,SAChB,QAAO;WACE,KAAK,SAAS,qBACvB,QAAO,2BAA2B,KAAK;EAEzC,MAAM,aAAa,EAAE;AACrB,OAAK,MAAM,KAAK,KAAK,WACnB,YAAW,KAAK,SAAS,EAAE,CAAC;AAE9B,SAAO;GACL,MAAM,KAAK;GACX;GACD;;AAOH,QAJe;EACb,GAAG;EACH,OAAO,SAAS,MAAM;EACvB;;AAIH,SAAgB,2CACd,MACS;AACT,KAAI,KAAK,SAAS,qBAChB,QAAO,CAAC,CAAC,KAAK;AAEhB,KAAI,KAAK,SAAS,SAAS,KAAK,SAAS,KACvC,QAAO,KAAK,WAAW,MAAK,MAC1B,2CAA2C,EAAE,CAC9C;AAGH,QAAO;;AAGT,SAAgB,kBACd,YACA,WACA;CACA,MAAM,UAAuB,EAAE;CAC/B,MAAM,aAA0B,EAAE;AAClC,MAAK,MAAM,KAAK,WACd,KAAI,UAAU,EAAE,CACd,SAAQ,KAAK,EAAE;KAEf,YAAW,KAAK,EAAE;AAGtB,QAAO,CAAC,SAAS,WAAW"}
|
|
@@ -4,10 +4,10 @@ import { once, toSorted } from "../../../shared/src/iterables.js";
|
|
|
4
4
|
import { must } from "../../../shared/src/must.js";
|
|
5
5
|
import { skipYields } from "./skip-yields.js";
|
|
6
6
|
import { makeAddChange, makeEditChange, makeRemoveChange } from "./change.js";
|
|
7
|
-
import { assertOrderingIncludesPK } from "../query/complete-ordering.js";
|
|
8
7
|
import { compareValues, makeComparator, valuesEqual } from "./data.js";
|
|
9
8
|
import { filterPush } from "./filter-push.js";
|
|
10
9
|
import { constraintMatchesPrimaryKey, constraintMatchesRow, primaryKeyConstraintFromFilters } from "./constraint.js";
|
|
10
|
+
import { assertOrderingIncludesPK } from "../query/complete-ordering.js";
|
|
11
11
|
import { createPredicate, transformFilters } from "../builder/filter.js";
|
|
12
12
|
import { BTreeSet } from "../../../shared/src/btree-set.js";
|
|
13
13
|
import { makeSourceChangeAdd, makeSourceChangeRemove } from "./source.js";
|
package/out/zql/src/ivm/take.js
CHANGED
|
@@ -2,8 +2,8 @@ import { assert, unreachable } from "../../../shared/src/asserts.js";
|
|
|
2
2
|
import { hasOwn } from "../../../shared/src/has-own.js";
|
|
3
3
|
import { throwOutput } from "./operator.js";
|
|
4
4
|
import { makeAddChange, makeRemoveChange } from "./change.js";
|
|
5
|
-
import { assertOrderingIncludesPK } from "../query/complete-ordering.js";
|
|
6
5
|
import { compareValues } from "./data.js";
|
|
6
|
+
import { assertOrderingIncludesPK } from "../query/complete-ordering.js";
|
|
7
7
|
//#region ../zql/src/ivm/take.ts
|
|
8
8
|
var MAX_BOUND_KEY = "maxBound";
|
|
9
9
|
/**
|
|
@@ -405,6 +405,6 @@ function makePartitionKeyComparator(partitionKey) {
|
|
|
405
405
|
};
|
|
406
406
|
}
|
|
407
407
|
//#endregion
|
|
408
|
-
export { Take
|
|
408
|
+
export { Take };
|
|
409
409
|
|
|
410
410
|
//# sourceMappingURL=take.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { assert, unreachable } from "../../shared/src/asserts.js";
|
|
2
2
|
import { must } from "../../shared/src/must.js";
|
|
3
|
-
import { assertOrderingIncludesPK } from "../../zql/src/query/complete-ordering.js";
|
|
4
3
|
import { makeComparator } from "../../zql/src/ivm/data.js";
|
|
4
|
+
import { assertOrderingIncludesPK } from "../../zql/src/query/complete-ordering.js";
|
|
5
5
|
import { createPredicate, transformFilters } from "../../zql/src/builder/filter.js";
|
|
6
6
|
import { genPushAndWriteWithSplitEdit, generateWithOverlay, generateWithOverlayUnordered, generateWithStart } from "../../zql/src/ivm/memory-source.js";
|
|
7
7
|
import { timeSampled } from "../../otel/src/maybe-time.js";
|
package/package.json
CHANGED
package/out/zql/src/ivm/cap.d.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { type Change } from './change.ts';
|
|
2
|
-
import type { Node } from './data.ts';
|
|
3
|
-
import { type FetchRequest, type Input, type Operator, type Output, type Storage } from './operator.ts';
|
|
4
|
-
import type { SourceSchema } from './schema.ts';
|
|
5
|
-
import { type Stream } from './stream.ts';
|
|
6
|
-
import { type PartitionKey } from './take.ts';
|
|
7
|
-
/**
|
|
8
|
-
* The Cap operator is a count-based limiter for EXISTS subqueries that
|
|
9
|
-
* does not require ordering. Unlike Take, it tracks membership by primary
|
|
10
|
-
* key set rather than by a sorted bound. This means:
|
|
11
|
-
*
|
|
12
|
-
* - No comparator needed (no ordering requirement)
|
|
13
|
-
* - No `start` or `reverse` fetch support
|
|
14
|
-
* - No `#rowHiddenFromFetch` complexity (we can defer when adding to the pk set)
|
|
15
|
-
*
|
|
16
|
-
* Cap is used in EXISTS child pipelines where only the count of matching
|
|
17
|
-
* rows matters, not their order. This allows SQLite to skip ORDER BY
|
|
18
|
-
* entirely, enabling much faster query plans.
|
|
19
|
-
*
|
|
20
|
-
* Cap can count rows globally or by unique value of some partition key
|
|
21
|
-
* (same as Take).
|
|
22
|
-
*/
|
|
23
|
-
export declare class Cap implements Operator {
|
|
24
|
-
#private;
|
|
25
|
-
constructor(input: Input, storage: Storage, limit: number, partitionKey?: PartitionKey);
|
|
26
|
-
setOutput(output: Output): void;
|
|
27
|
-
getSchema(): SourceSchema;
|
|
28
|
-
fetch(req: FetchRequest): Stream<Node | 'yield'>;
|
|
29
|
-
push(change: Change): Stream<'yield'>;
|
|
30
|
-
destroy(): void;
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=cap.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cap.d.ts","sourceRoot":"","sources":["../../../../../zql/src/ivm/cap.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,KAAK,MAAM,EAAkB,MAAM,aAAa,CAAC;AAExE,OAAO,KAAK,EAAa,IAAI,EAAC,MAAM,WAAW,CAAC;AAChD,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,MAAM,EACX,KAAK,OAAO,EACb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,aAAa,CAAC;AACxC,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,WAAW,CAAC;AAanB;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,GAAI,YAAW,QAAQ;;gBAWhC,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,YAAY;IAa7B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI/B,SAAS,IAAI,YAAY;IAIxB,KAAK,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC;IA0FhD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;IAsHtC,OAAO,IAAI,IAAI;CAGhB"}
|
package/out/zql/src/ivm/cap.js
DELETED
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
import { assert } from "../../../shared/src/asserts.js";
|
|
2
|
-
import { throwOutput } from "./operator.js";
|
|
3
|
-
import { makeAddChange } from "./change.js";
|
|
4
|
-
import { constraintMatchesPartitionKey, makePartitionKeyComparator } from "./take.js";
|
|
5
|
-
//#region ../zql/src/ivm/cap.ts
|
|
6
|
-
/**
|
|
7
|
-
* The Cap operator is a count-based limiter for EXISTS subqueries that
|
|
8
|
-
* does not require ordering. Unlike Take, it tracks membership by primary
|
|
9
|
-
* key set rather than by a sorted bound. This means:
|
|
10
|
-
*
|
|
11
|
-
* - No comparator needed (no ordering requirement)
|
|
12
|
-
* - No `start` or `reverse` fetch support
|
|
13
|
-
* - No `#rowHiddenFromFetch` complexity (we can defer when adding to the pk set)
|
|
14
|
-
*
|
|
15
|
-
* Cap is used in EXISTS child pipelines where only the count of matching
|
|
16
|
-
* rows matters, not their order. This allows SQLite to skip ORDER BY
|
|
17
|
-
* entirely, enabling much faster query plans.
|
|
18
|
-
*
|
|
19
|
-
* Cap can count rows globally or by unique value of some partition key
|
|
20
|
-
* (same as Take).
|
|
21
|
-
*/
|
|
22
|
-
var Cap = class {
|
|
23
|
-
#input;
|
|
24
|
-
#storage;
|
|
25
|
-
#limit;
|
|
26
|
-
#partitionKey;
|
|
27
|
-
#partitionKeyComparator;
|
|
28
|
-
#primaryKey;
|
|
29
|
-
#output = throwOutput;
|
|
30
|
-
constructor(input, storage, limit, partitionKey) {
|
|
31
|
-
assert(limit >= 0, "Limit must be non-negative");
|
|
32
|
-
input.setOutput(this);
|
|
33
|
-
this.#input = input;
|
|
34
|
-
this.#storage = storage;
|
|
35
|
-
this.#limit = limit;
|
|
36
|
-
this.#partitionKey = partitionKey;
|
|
37
|
-
this.#partitionKeyComparator = partitionKey && makePartitionKeyComparator(partitionKey);
|
|
38
|
-
this.#primaryKey = input.getSchema().primaryKey;
|
|
39
|
-
}
|
|
40
|
-
setOutput(output) {
|
|
41
|
-
this.#output = output;
|
|
42
|
-
}
|
|
43
|
-
getSchema() {
|
|
44
|
-
return this.#input.getSchema();
|
|
45
|
-
}
|
|
46
|
-
*fetch(req) {
|
|
47
|
-
assert(!req.start, "Cap does not support start");
|
|
48
|
-
assert(!req.reverse, "Cap does not support reverse");
|
|
49
|
-
assert(!this.#partitionKey || req.constraint !== void 0 && constraintMatchesPartitionKey(req.constraint, this.#partitionKey), "Cap fetch: constraint must match partition key when partitioned");
|
|
50
|
-
const capStateKey = getCapStateKey(this.#partitionKey, req.constraint);
|
|
51
|
-
const capState = this.#storage.get(capStateKey);
|
|
52
|
-
if (!capState) {
|
|
53
|
-
yield* this.#initialFetch(req);
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
if (capState.size === 0) return;
|
|
57
|
-
for (const pk of capState.pks) {
|
|
58
|
-
const constraint = deserializePKToConstraint(pk, this.#primaryKey);
|
|
59
|
-
for (const inputNode of this.#input.fetch({ constraint })) {
|
|
60
|
-
if (inputNode === "yield") {
|
|
61
|
-
yield inputNode;
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
yield inputNode;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
*#initialFetch(req) {
|
|
69
|
-
if (this.#limit === 0) return;
|
|
70
|
-
assert(constraintMatchesPartitionKey(req.constraint, this.#partitionKey), "Constraint should match partition key");
|
|
71
|
-
const capStateKey = getCapStateKey(this.#partitionKey, req.constraint);
|
|
72
|
-
assert(this.#storage.get(capStateKey) === void 0, "Cap state should be undefined");
|
|
73
|
-
let size = 0;
|
|
74
|
-
const pks = [];
|
|
75
|
-
let downstreamEarlyReturn = true;
|
|
76
|
-
let exceptionThrown = false;
|
|
77
|
-
try {
|
|
78
|
-
for (const inputNode of this.#input.fetch(req)) {
|
|
79
|
-
if (inputNode === "yield") {
|
|
80
|
-
yield "yield";
|
|
81
|
-
continue;
|
|
82
|
-
}
|
|
83
|
-
yield inputNode;
|
|
84
|
-
pks.push(serializePK(inputNode.row, this.#primaryKey));
|
|
85
|
-
size++;
|
|
86
|
-
if (size === this.#limit) break;
|
|
87
|
-
}
|
|
88
|
-
downstreamEarlyReturn = false;
|
|
89
|
-
} catch (e) {
|
|
90
|
-
exceptionThrown = true;
|
|
91
|
-
throw e;
|
|
92
|
-
} finally {
|
|
93
|
-
if (!exceptionThrown) {
|
|
94
|
-
this.#storage.set(capStateKey, {
|
|
95
|
-
size,
|
|
96
|
-
pks
|
|
97
|
-
});
|
|
98
|
-
assert(!downstreamEarlyReturn, "Unexpected early return prevented full hydration");
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
*push(change) {
|
|
103
|
-
if (change[0] === 2) {
|
|
104
|
-
yield* this.#pushEditChange(change);
|
|
105
|
-
return;
|
|
106
|
-
}
|
|
107
|
-
const capStateKey = getCapStateKey(this.#partitionKey, change[1].row);
|
|
108
|
-
const capState = this.#storage.get(capStateKey);
|
|
109
|
-
if (!capState) return;
|
|
110
|
-
const pk = serializePK(change[1].row, this.#primaryKey);
|
|
111
|
-
if (change[0] === 0) {
|
|
112
|
-
if (capState.size < this.#limit) {
|
|
113
|
-
const pks = [...capState.pks, pk];
|
|
114
|
-
this.#storage.set(capStateKey, {
|
|
115
|
-
size: capState.size + 1,
|
|
116
|
-
pks
|
|
117
|
-
});
|
|
118
|
-
yield* this.#output.push(change, this);
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
return;
|
|
122
|
-
} else if (change[0] === 1) {
|
|
123
|
-
const pkIndex = capState.pks.indexOf(pk);
|
|
124
|
-
if (pkIndex === -1) return;
|
|
125
|
-
const pks = [...capState.pks];
|
|
126
|
-
pks.splice(pkIndex, 1);
|
|
127
|
-
const newSize = capState.size - 1;
|
|
128
|
-
const pkSet = new Set(pks);
|
|
129
|
-
const constraint = this.#partitionKey ? Object.fromEntries(this.#partitionKey.map((key) => [key, change[1].row[key]])) : void 0;
|
|
130
|
-
let replacement;
|
|
131
|
-
for (const node of this.#input.fetch({ constraint })) {
|
|
132
|
-
if (node === "yield") {
|
|
133
|
-
yield node;
|
|
134
|
-
continue;
|
|
135
|
-
}
|
|
136
|
-
const nodePK = serializePK(node.row, this.#primaryKey);
|
|
137
|
-
if (!pkSet.has(nodePK)) {
|
|
138
|
-
replacement = node;
|
|
139
|
-
break;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
if (replacement) {
|
|
143
|
-
this.#storage.set(capStateKey, {
|
|
144
|
-
size: newSize,
|
|
145
|
-
pks
|
|
146
|
-
});
|
|
147
|
-
yield* this.#output.push(change, this);
|
|
148
|
-
const replacementPK = serializePK(replacement.row, this.#primaryKey);
|
|
149
|
-
pks.push(replacementPK);
|
|
150
|
-
this.#storage.set(capStateKey, {
|
|
151
|
-
size: newSize + 1,
|
|
152
|
-
pks
|
|
153
|
-
});
|
|
154
|
-
yield* this.#output.push(makeAddChange(replacement), this);
|
|
155
|
-
} else {
|
|
156
|
-
this.#storage.set(capStateKey, {
|
|
157
|
-
size: newSize,
|
|
158
|
-
pks
|
|
159
|
-
});
|
|
160
|
-
yield* this.#output.push(change, this);
|
|
161
|
-
}
|
|
162
|
-
} else if (change[0] === 3) {
|
|
163
|
-
if (new Set(capState.pks).has(pk)) yield* this.#output.push(change, this);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
*#pushEditChange(change) {
|
|
167
|
-
assert(!this.#partitionKeyComparator || this.#partitionKeyComparator(change[2].row, change[1].row) === 0, "Unexpected change of partition key");
|
|
168
|
-
const capStateKey = getCapStateKey(this.#partitionKey, change[2].row);
|
|
169
|
-
const capState = this.#storage.get(capStateKey);
|
|
170
|
-
if (!capState) return;
|
|
171
|
-
const oldPK = serializePK(change[2].row, this.#primaryKey);
|
|
172
|
-
if (new Set(capState.pks).has(oldPK)) {
|
|
173
|
-
const newPK = serializePK(change[1].row, this.#primaryKey);
|
|
174
|
-
if (oldPK !== newPK) {
|
|
175
|
-
const pks = capState.pks.map((p) => p === oldPK ? newPK : p);
|
|
176
|
-
this.#storage.set(capStateKey, {
|
|
177
|
-
size: capState.size,
|
|
178
|
-
pks
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
yield* this.#output.push(change, this);
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
destroy() {
|
|
185
|
-
this.#input.destroy();
|
|
186
|
-
}
|
|
187
|
-
};
|
|
188
|
-
function getCapStateKey(partitionKey, rowOrConstraint) {
|
|
189
|
-
const partitionValues = [];
|
|
190
|
-
if (partitionKey && rowOrConstraint) for (const key of partitionKey) partitionValues.push(rowOrConstraint[key]);
|
|
191
|
-
return JSON.stringify(["cap", ...partitionValues]);
|
|
192
|
-
}
|
|
193
|
-
function serializePK(row, primaryKey) {
|
|
194
|
-
return JSON.stringify(primaryKey.map((k) => row[k]));
|
|
195
|
-
}
|
|
196
|
-
function deserializePKToConstraint(pk, primaryKey) {
|
|
197
|
-
const values = JSON.parse(pk);
|
|
198
|
-
const constraint = {};
|
|
199
|
-
for (let i = 0; i < primaryKey.length; i++) constraint[primaryKey[i]] = values[i];
|
|
200
|
-
return constraint;
|
|
201
|
-
}
|
|
202
|
-
//#endregion
|
|
203
|
-
export { Cap };
|
|
204
|
-
|
|
205
|
-
//# sourceMappingURL=cap.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cap.js","names":["#input","#storage","#limit","#partitionKey","#partitionKeyComparator","#primaryKey","#output","#initialFetch","#pushEditChange"],"sources":["../../../../../zql/src/ivm/cap.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {Row, Value} from '../../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../../zero-protocol/src/primary-key.ts';\nimport {ChangeIndex} from './change-index.ts';\nimport {ChangeType} from './change-type.ts';\nimport {makeAddChange, type Change, type EditChange} from './change.ts';\nimport type {Constraint} from './constraint.ts';\nimport type {Comparator, Node} from './data.ts';\nimport {\n throwOutput,\n type FetchRequest,\n type Input,\n type Operator,\n type Output,\n type Storage,\n} from './operator.ts';\nimport type {SourceSchema} from './schema.ts';\nimport {type Stream} from './stream.ts';\nimport {\n constraintMatchesPartitionKey,\n makePartitionKeyComparator,\n type PartitionKey,\n} from './take.ts';\n\ntype CapState = {\n size: number;\n pks: string[];\n};\n\ninterface CapStorage {\n get(key: string): CapState | undefined;\n set(key: string, value: CapState): void;\n del(key: string): void;\n}\n\n/**\n * The Cap operator is a count-based limiter for EXISTS subqueries that\n * does not require ordering. Unlike Take, it tracks membership by primary\n * key set rather than by a sorted bound. This means:\n *\n * - No comparator needed (no ordering requirement)\n * - No `start` or `reverse` fetch support\n * - No `#rowHiddenFromFetch` complexity (we can defer when adding to the pk set)\n *\n * Cap is used in EXISTS child pipelines where only the count of matching\n * rows matters, not their order. This allows SQLite to skip ORDER BY\n * entirely, enabling much faster query plans.\n *\n * Cap can count rows globally or by unique value of some partition key\n * (same as Take).\n */\nexport class Cap implements Operator {\n readonly #input: Input;\n readonly #storage: CapStorage;\n readonly #limit: number;\n readonly #partitionKey: PartitionKey | undefined;\n readonly #partitionKeyComparator: Comparator | undefined;\n readonly #primaryKey: PrimaryKey;\n\n #output: Output = throwOutput;\n\n constructor(\n input: Input,\n storage: Storage,\n limit: number,\n partitionKey?: PartitionKey,\n ) {\n assert(limit >= 0, 'Limit must be non-negative');\n input.setOutput(this);\n this.#input = input;\n this.#storage = storage as CapStorage;\n this.#limit = limit;\n this.#partitionKey = partitionKey;\n this.#partitionKeyComparator =\n partitionKey && makePartitionKeyComparator(partitionKey);\n this.#primaryKey = input.getSchema().primaryKey;\n }\n\n setOutput(output: Output): void {\n this.#output = output;\n }\n\n getSchema(): SourceSchema {\n return this.#input.getSchema();\n }\n\n *fetch(req: FetchRequest): Stream<Node | 'yield'> {\n assert(!req.start, 'Cap does not support start');\n assert(!req.reverse, 'Cap does not support reverse');\n\n // Cap is only built for non-flipped EXISTS subqueries, whose only\n // downstream consumer is a Join that always fetches with a constraint\n // built from the correlation's childField — which is Cap's partition\n // key. So either partitionKey is undefined, or constraint matches.\n assert(\n !this.#partitionKey ||\n (req.constraint !== undefined &&\n constraintMatchesPartitionKey(req.constraint, this.#partitionKey)),\n 'Cap fetch: constraint must match partition key when partitioned',\n );\n\n const capStateKey = getCapStateKey(this.#partitionKey, req.constraint);\n const capState = this.#storage.get(capStateKey);\n if (!capState) {\n yield* this.#initialFetch(req);\n return;\n }\n if (capState.size === 0) {\n return;\n }\n // PK-based point lookups: fetch each tracked row by its PK directly,\n // rather than scanning the partition and filtering.\n for (const pk of capState.pks) {\n const constraint = deserializePKToConstraint(pk, this.#primaryKey);\n for (const inputNode of this.#input.fetch({constraint})) {\n if (inputNode === 'yield') {\n yield inputNode;\n continue;\n }\n yield inputNode;\n }\n }\n }\n\n *#initialFetch(req: FetchRequest): Stream<Node | 'yield'> {\n if (this.#limit === 0) {\n return;\n }\n\n assert(\n constraintMatchesPartitionKey(req.constraint, this.#partitionKey),\n 'Constraint should match partition key',\n );\n\n const capStateKey = getCapStateKey(this.#partitionKey, req.constraint);\n assert(\n this.#storage.get(capStateKey) === undefined,\n 'Cap state should be undefined',\n );\n\n let size = 0;\n const pks: string[] = [];\n let downstreamEarlyReturn = true;\n let exceptionThrown = false;\n try {\n for (const inputNode of this.#input.fetch(req)) {\n if (inputNode === 'yield') {\n yield 'yield';\n continue;\n }\n yield inputNode;\n pks.push(serializePK(inputNode.row, this.#primaryKey));\n size++;\n if (size === this.#limit) {\n break;\n }\n }\n downstreamEarlyReturn = false;\n } catch (e) {\n exceptionThrown = true;\n throw e;\n } finally {\n if (!exceptionThrown) {\n this.#storage.set(capStateKey, {size, pks});\n // If it becomes necessary to support downstream early return, this\n // assert should be removed, and replaced with code that consumes\n // the input stream until limit is reached or the input stream is\n // exhausted so that capState is properly hydrated.\n assert(\n !downstreamEarlyReturn,\n 'Unexpected early return prevented full hydration',\n );\n }\n }\n }\n\n *push(change: Change): Stream<'yield'> {\n if (change[ChangeIndex.TYPE] === ChangeType.EDIT) {\n yield* this.#pushEditChange(change);\n return;\n }\n\n const capStateKey = getCapStateKey(\n this.#partitionKey,\n change[ChangeIndex.NODE].row,\n );\n const capState = this.#storage.get(capStateKey);\n if (!capState) {\n return;\n }\n\n const pk = serializePK(change[ChangeIndex.NODE].row, this.#primaryKey);\n\n if (change[ChangeIndex.TYPE] === ChangeType.ADD) {\n if (capState.size < this.#limit) {\n const pks = [...capState.pks, pk];\n this.#storage.set(capStateKey, {size: capState.size + 1, pks});\n yield* this.#output.push(change, this);\n return;\n }\n // Full — drop\n return;\n } else if (change[ChangeIndex.TYPE] === ChangeType.REMOVE) {\n const pkIndex = capState.pks.indexOf(pk);\n if (pkIndex === -1) {\n // Not in our set — drop\n return;\n }\n // Remove from set\n const pks = [...capState.pks];\n pks.splice(pkIndex, 1);\n const newSize = capState.size - 1;\n\n // Try to refill: fetch from input with partition constraint,\n // find first row NOT in PK set\n const pkSet = new Set(pks);\n const constraint = this.#partitionKey\n ? (Object.fromEntries(\n this.#partitionKey.map(\n key => [key, change[ChangeIndex.NODE].row[key]] as const,\n ),\n ) as Constraint)\n : undefined;\n\n let replacement: Node | undefined;\n for (const node of this.#input.fetch({constraint})) {\n if (node === 'yield') {\n yield node;\n continue;\n }\n const nodePK = serializePK(node.row, this.#primaryKey);\n if (!pkSet.has(nodePK)) {\n replacement = node;\n break;\n }\n }\n\n if (replacement) {\n // Store state WITHOUT replacement during remove forward,\n // matching Take's pattern of hiding in-flight changes from re-fetches.\n this.#storage.set(capStateKey, {size: newSize, pks});\n yield* this.#output.push(change, this);\n // Now add replacement to set and forward the add.\n const replacementPK = serializePK(replacement.row, this.#primaryKey);\n pks.push(replacementPK);\n this.#storage.set(capStateKey, {size: newSize + 1, pks});\n yield* this.#output.push(makeAddChange(replacement), this);\n } else {\n this.#storage.set(capStateKey, {size: newSize, pks});\n yield* this.#output.push(change, this);\n }\n } else if (change[ChangeIndex.TYPE] === ChangeType.CHILD) {\n const pkSet = new Set(capState.pks);\n if (pkSet.has(pk)) {\n yield* this.#output.push(change, this);\n }\n }\n }\n\n *#pushEditChange(change: EditChange): Stream<'yield'> {\n assert(\n !this.#partitionKeyComparator ||\n this.#partitionKeyComparator(\n change[ChangeIndex.OLD_NODE].row,\n change[ChangeIndex.NODE].row,\n ) === 0,\n 'Unexpected change of partition key',\n );\n const capStateKey = getCapStateKey(\n this.#partitionKey,\n change[ChangeIndex.OLD_NODE].row,\n );\n const capState = this.#storage.get(capStateKey);\n if (!capState) {\n return;\n }\n\n const oldPK = serializePK(\n change[ChangeIndex.OLD_NODE].row,\n this.#primaryKey,\n );\n const pkSet = new Set(capState.pks);\n if (pkSet.has(oldPK)) {\n // Update the PK in our set if it changed\n const newPK = serializePK(change[ChangeIndex.NODE].row, this.#primaryKey);\n if (oldPK !== newPK) {\n const pks = capState.pks.map(p => (p === oldPK ? newPK : p));\n this.#storage.set(capStateKey, {size: capState.size, pks});\n }\n yield* this.#output.push(change, this);\n }\n // If not in our set, drop\n }\n\n destroy(): void {\n this.#input.destroy();\n }\n}\n\nfunction getCapStateKey(\n partitionKey: PartitionKey | undefined,\n rowOrConstraint: Row | Constraint | undefined,\n): string {\n const partitionValues: Value[] = [];\n\n if (partitionKey && rowOrConstraint) {\n for (const key of partitionKey) {\n partitionValues.push(rowOrConstraint[key]);\n }\n }\n\n return JSON.stringify(['cap', ...partitionValues]);\n}\n\nfunction serializePK(row: Row, primaryKey: PrimaryKey): string {\n return JSON.stringify(primaryKey.map(k => row[k]));\n}\n\nfunction deserializePKToConstraint(\n pk: string,\n primaryKey: PrimaryKey,\n): Constraint {\n const values = JSON.parse(pk) as Value[];\n const constraint: Record<string, Value> = {};\n for (let i = 0; i < primaryKey.length; i++) {\n constraint[primaryKey[i]] = values[i];\n }\n return constraint;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAmDA,IAAa,MAAb,MAAqC;CACnC;CACA;CACA;CACA;CACA;CACA;CAEA,UAAkB;CAElB,YACE,OACA,SACA,OACA,cACA;AACA,SAAO,SAAS,GAAG,6BAA6B;AAChD,QAAM,UAAU,KAAK;AACrB,QAAA,QAAc;AACd,QAAA,UAAgB;AAChB,QAAA,QAAc;AACd,QAAA,eAAqB;AACrB,QAAA,yBACE,gBAAgB,2BAA2B,aAAa;AAC1D,QAAA,aAAmB,MAAM,WAAW,CAAC;;CAGvC,UAAU,QAAsB;AAC9B,QAAA,SAAe;;CAGjB,YAA0B;AACxB,SAAO,MAAA,MAAY,WAAW;;CAGhC,CAAC,MAAM,KAA2C;AAChD,SAAO,CAAC,IAAI,OAAO,6BAA6B;AAChD,SAAO,CAAC,IAAI,SAAS,+BAA+B;AAMpD,SACE,CAAC,MAAA,gBACE,IAAI,eAAe,KAAA,KAClB,8BAA8B,IAAI,YAAY,MAAA,aAAmB,EACrE,kEACD;EAED,MAAM,cAAc,eAAe,MAAA,cAAoB,IAAI,WAAW;EACtE,MAAM,WAAW,MAAA,QAAc,IAAI,YAAY;AAC/C,MAAI,CAAC,UAAU;AACb,UAAO,MAAA,aAAmB,IAAI;AAC9B;;AAEF,MAAI,SAAS,SAAS,EACpB;AAIF,OAAK,MAAM,MAAM,SAAS,KAAK;GAC7B,MAAM,aAAa,0BAA0B,IAAI,MAAA,WAAiB;AAClE,QAAK,MAAM,aAAa,MAAA,MAAY,MAAM,EAAC,YAAW,CAAC,EAAE;AACvD,QAAI,cAAc,SAAS;AACzB,WAAM;AACN;;AAEF,UAAM;;;;CAKZ,EAAA,aAAe,KAA2C;AACxD,MAAI,MAAA,UAAgB,EAClB;AAGF,SACE,8BAA8B,IAAI,YAAY,MAAA,aAAmB,EACjE,wCACD;EAED,MAAM,cAAc,eAAe,MAAA,cAAoB,IAAI,WAAW;AACtE,SACE,MAAA,QAAc,IAAI,YAAY,KAAK,KAAA,GACnC,gCACD;EAED,IAAI,OAAO;EACX,MAAM,MAAgB,EAAE;EACxB,IAAI,wBAAwB;EAC5B,IAAI,kBAAkB;AACtB,MAAI;AACF,QAAK,MAAM,aAAa,MAAA,MAAY,MAAM,IAAI,EAAE;AAC9C,QAAI,cAAc,SAAS;AACzB,WAAM;AACN;;AAEF,UAAM;AACN,QAAI,KAAK,YAAY,UAAU,KAAK,MAAA,WAAiB,CAAC;AACtD;AACA,QAAI,SAAS,MAAA,MACX;;AAGJ,2BAAwB;WACjB,GAAG;AACV,qBAAkB;AAClB,SAAM;YACE;AACR,OAAI,CAAC,iBAAiB;AACpB,UAAA,QAAc,IAAI,aAAa;KAAC;KAAM;KAAI,CAAC;AAK3C,WACE,CAAC,uBACD,mDACD;;;;CAKP,CAAC,KAAK,QAAiC;AACrC,MAAI,OAAO,OAAsB,GAAiB;AAChD,UAAO,MAAA,eAAqB,OAAO;AACnC;;EAGF,MAAM,cAAc,eAClB,MAAA,cACA,OAAO,GAAkB,IAC1B;EACD,MAAM,WAAW,MAAA,QAAc,IAAI,YAAY;AAC/C,MAAI,CAAC,SACH;EAGF,MAAM,KAAK,YAAY,OAAO,GAAkB,KAAK,MAAA,WAAiB;AAEtE,MAAI,OAAO,OAAsB,GAAgB;AAC/C,OAAI,SAAS,OAAO,MAAA,OAAa;IAC/B,MAAM,MAAM,CAAC,GAAG,SAAS,KAAK,GAAG;AACjC,UAAA,QAAc,IAAI,aAAa;KAAC,MAAM,SAAS,OAAO;KAAG;KAAI,CAAC;AAC9D,WAAO,MAAA,OAAa,KAAK,QAAQ,KAAK;AACtC;;AAGF;aACS,OAAO,OAAsB,GAAmB;GACzD,MAAM,UAAU,SAAS,IAAI,QAAQ,GAAG;AACxC,OAAI,YAAY,GAEd;GAGF,MAAM,MAAM,CAAC,GAAG,SAAS,IAAI;AAC7B,OAAI,OAAO,SAAS,EAAE;GACtB,MAAM,UAAU,SAAS,OAAO;GAIhC,MAAM,QAAQ,IAAI,IAAI,IAAI;GAC1B,MAAM,aAAa,MAAA,eACd,OAAO,YACN,MAAA,aAAmB,KACjB,QAAO,CAAC,KAAK,OAAO,GAAkB,IAAI,KAAK,CAChD,CACF,GACD,KAAA;GAEJ,IAAI;AACJ,QAAK,MAAM,QAAQ,MAAA,MAAY,MAAM,EAAC,YAAW,CAAC,EAAE;AAClD,QAAI,SAAS,SAAS;AACpB,WAAM;AACN;;IAEF,MAAM,SAAS,YAAY,KAAK,KAAK,MAAA,WAAiB;AACtD,QAAI,CAAC,MAAM,IAAI,OAAO,EAAE;AACtB,mBAAc;AACd;;;AAIJ,OAAI,aAAa;AAGf,UAAA,QAAc,IAAI,aAAa;KAAC,MAAM;KAAS;KAAI,CAAC;AACpD,WAAO,MAAA,OAAa,KAAK,QAAQ,KAAK;IAEtC,MAAM,gBAAgB,YAAY,YAAY,KAAK,MAAA,WAAiB;AACpE,QAAI,KAAK,cAAc;AACvB,UAAA,QAAc,IAAI,aAAa;KAAC,MAAM,UAAU;KAAG;KAAI,CAAC;AACxD,WAAO,MAAA,OAAa,KAAK,cAAc,YAAY,EAAE,KAAK;UACrD;AACL,UAAA,QAAc,IAAI,aAAa;KAAC,MAAM;KAAS;KAAI,CAAC;AACpD,WAAO,MAAA,OAAa,KAAK,QAAQ,KAAK;;aAE/B,OAAO,OAAsB;OACxB,IAAI,IAAI,SAAS,IAAI,CACzB,IAAI,GAAG,CACf,QAAO,MAAA,OAAa,KAAK,QAAQ,KAAK;;;CAK5C,EAAA,eAAiB,QAAqC;AACpD,SACE,CAAC,MAAA,0BACC,MAAA,uBACE,OAAO,GAAsB,KAC7B,OAAO,GAAkB,IAC1B,KAAK,GACR,qCACD;EACD,MAAM,cAAc,eAClB,MAAA,cACA,OAAO,GAAsB,IAC9B;EACD,MAAM,WAAW,MAAA,QAAc,IAAI,YAAY;AAC/C,MAAI,CAAC,SACH;EAGF,MAAM,QAAQ,YACZ,OAAO,GAAsB,KAC7B,MAAA,WACD;AAED,MADc,IAAI,IAAI,SAAS,IAAI,CACzB,IAAI,MAAM,EAAE;GAEpB,MAAM,QAAQ,YAAY,OAAO,GAAkB,KAAK,MAAA,WAAiB;AACzE,OAAI,UAAU,OAAO;IACnB,MAAM,MAAM,SAAS,IAAI,KAAI,MAAM,MAAM,QAAQ,QAAQ,EAAG;AAC5D,UAAA,QAAc,IAAI,aAAa;KAAC,MAAM,SAAS;KAAM;KAAI,CAAC;;AAE5D,UAAO,MAAA,OAAa,KAAK,QAAQ,KAAK;;;CAK1C,UAAgB;AACd,QAAA,MAAY,SAAS;;;AAIzB,SAAS,eACP,cACA,iBACQ;CACR,MAAM,kBAA2B,EAAE;AAEnC,KAAI,gBAAgB,gBAClB,MAAK,MAAM,OAAO,aAChB,iBAAgB,KAAK,gBAAgB,KAAK;AAI9C,QAAO,KAAK,UAAU,CAAC,OAAO,GAAG,gBAAgB,CAAC;;AAGpD,SAAS,YAAY,KAAU,YAAgC;AAC7D,QAAO,KAAK,UAAU,WAAW,KAAI,MAAK,IAAI,GAAG,CAAC;;AAGpD,SAAS,0BACP,IACA,YACY;CACZ,MAAM,SAAS,KAAK,MAAM,GAAG;CAC7B,MAAM,aAAoC,EAAE;AAC5C,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,IACrC,YAAW,WAAW,MAAM,OAAO;AAErC,QAAO"}
|