npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.1 → 1.10.0-canary.3 - Mend

@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (180) hide show

package/out/zqlite/src/table-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"table-source.js","names":["#dbCache","#connections","#table","#columns","#uniqueIndexes","#primaryKey","#logConfig","#lc","#shouldYield","#stmts","#getStatementsFor","#allColumns","#fetch","#getSchema","#requestToSQL","#mapFromSQLiteTypes","#overlay","#writeChange","#pushEpoch","#getRowStmtCache","#getRowStmt"],"sources":["../../../../zqlite/src/table-source.ts"],"sourcesContent":["import type {SQLQuery} from '@databases/sql';\nimport type {LogContext} from '@rocicorp/logger';\nimport SQLite3Database from '@rocicorp/zero-sqlite3';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {timeSampled} from '../../otel/src/maybe-time.ts';\nimport {assert, unreachable} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Writable} from '../../shared/src/writable.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {Row, Value} from '../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../zero-protocol/src/primary-key.ts';\nimport type {\n SchemaValue,\n ValueType,\n} from '../../zero-schema/src/table-schema.ts';\nimport type {DebugDelegate} from '../../zql/src/builder/debug-delegate.ts';\nimport {\n createPredicate,\n transformFilters,\n} from '../../zql/src/builder/filter.ts';\nimport {ChangeType} from '../../zql/src/ivm/change-type.ts';\nimport {makeComparator, type Node} from '../../zql/src/ivm/data.ts';\nimport {\n generateWithOverlay,\n generateWithOverlayUnordered,\n generateWithStart,\n genPushAndWriteWithSplitEdit,\n type Connection,\n type Overlay,\n} from '../../zql/src/ivm/memory-source.ts';\nimport {type FetchRequest} from '../../zql/src/ivm/operator.ts';\nimport type {SourceSchema} from '../../zql/src/ivm/schema.ts';\nimport {SourceChangeIndex} from '../../zql/src/ivm/source-change-index.ts';\nimport {\n type Source,\n type SourceChange,\n type SourceInput,\n} from '../../zql/src/ivm/source.ts';\nimport type {Stream} from '../../zql/src/ivm/stream.ts';\nimport {assertOrderingIncludesPK} from '../../zql/src/query/complete-ordering.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compile, format, sql} from './internal/sql.ts';\nimport {StatementCache} from './internal/statement-cache.ts';\nimport {\n buildSelectQuery,\n toSQLiteType,\n type NoSubqueryCondition,\n} from './query-builder.ts';\n\ntype Statements = {\n readonly cache: StatementCache;\n readonly insert: Statement;\n readonly delete: Statement;\n readonly update: Statement \| undefined;\n readonly checkExists: Statement;\n readonly getExisting: Statement;\n};\n\nlet eventCount = 0;\n\n/*\n A source that is backed by a SQLite table.\n \n Values are written to the backing table _after_ being vended by the source.\n \n This ordering of events is to ensure self joins function properly. That is,\n * we can't reveal a value to an output before it has been pushed to that output.\n \n The code is fairly straightforward except for:\n * 1. Dealing with a `fetch` that has a basis of `before`.\n * 2. Dealing with compound orders that have differing directions (a ASC, b DESC, c ASC)\n \n See comments in relevant functions for more details.\n /\nexport class TableSource implements Source {\n readonly #dbCache = new WeakMap<Database, Statements>();\n readonly #connections: Connection[] = [];\n readonly #table: string;\n readonly #columns: Record<string, SchemaValue>;\n // Maps sorted columns JSON string (e.g. '[\"a\",\"b\"]) to Set of columns.\n readonly #uniqueIndexes: Map<string, Set<string>>;\n readonly #primaryKey: PrimaryKey;\n readonly #logConfig: LogConfig;\n readonly #lc: LogContext;\n readonly #shouldYield: () => boolean;\n #stmts: Statements;\n #overlay?: Overlay \| undefined;\n #pushEpoch = 0;\n\n /\n @param shouldYield a function called after each row is read from the database,\n * which should return true if the source should yield the special 'yield' value\n * to yield control back to the caller at the end of the pipeline. Can\n * also throw an error to abort the pipeline processing.\n /\n constructor(\n logContext: LogContext,\n logConfig: LogConfig,\n db: Database,\n tableName: string,\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n shouldYield = () => false,\n ) {\n this.#lc = logContext;\n this.#logConfig = logConfig;\n this.#table = tableName;\n this.#columns = columns;\n this.#uniqueIndexes = getUniqueIndexes(db, tableName);\n this.#primaryKey = primaryKey;\n this.#stmts = this.#getStatementsFor(db);\n this.#shouldYield = shouldYield;\n\n assert(\n this.#uniqueIndexes.has(JSON.stringify(primaryKey.toSorted())),\n `primary key ${primaryKey} does not have a UNIQUE index`,\n );\n }\n\n get tableSchema() {\n return {\n name: this.#table,\n columns: this.#columns,\n primaryKey: this.#primaryKey,\n };\n }\n\n /\n Sets the db (snapshot) to use, to facilitate the Snapshotter leapfrog\n * algorithm for concurrent traversal of historic timelines.\n /\n setDB(db: Database) {\n this.#stmts = this.#getStatementsFor(db);\n }\n\n #getStatementsFor(db: Database) {\n const cached = this.#dbCache.get(db);\n if (cached) {\n return cached;\n }\n\n const stmts = {\n cache: new StatementCache(db),\n insert: db.prepare(\n compile(\n sql`INSERT INTO ${sql.ident(this.#table)} (${sql.join(\n Object.keys(this.#columns).map(c => sql.ident(c)),\n ', ',\n )}) VALUES (${sql.__dangerous__rawValue(\n Array.from({length: Object.keys(this.#columns).length})\n .fill('?')\n .join(','),\n )})`,\n ),\n ),\n delete: db.prepare(\n compile(\n sql`DELETE FROM ${sql.ident(this.#table)} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n ),\n // If all the columns are part of the primary key, we cannot use UPDATE.\n update:\n Object.keys(this.#columns).length > this.#primaryKey.length\n ? db.prepare(\n compile(\n sql`UPDATE ${sql.ident(this.#table)} SET ${sql.join(\n nonPrimaryKeys(this.#columns, this.#primaryKey).map(\n c => sql`${sql.ident(c)}=?`,\n ),\n ',',\n )} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n )\n : undefined,\n checkExists: db.prepare(\n compile(\n sql`SELECT 1 AS \"exists\" FROM ${sql.ident(\n this.#table,\n )} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )} LIMIT 1`,\n ),\n ),\n getExisting: db.prepare(\n compile(\n sql`SELECT FROM ${sql.ident(this.#table)} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n ),\n };\n this.#dbCache.set(db, stmts);\n return stmts;\n }\n\n get #allColumns() {\n return sql.join(\n Object.keys(this.#columns).map(c => sql.ident(c)),\n sql`,`,\n );\n }\n\n #getSchema(connection: Connection, unordered: boolean): SourceSchema {\n return {\n tableName: this.#table,\n columns: this.#columns,\n primaryKey: this.#primaryKey,\n sort: unordered ? undefined : connection.sort,\n relationships: {},\n isHidden: false,\n system: 'client',\n compareRows: connection.compareRows,\n };\n }\n\n connect(\n sort: Ordering \| undefined,\n filters?: Condition,\n splitEditKeys?: Set<string>,\n debug?: DebugDelegate,\n ) {\n const transformedFilters = transformFilters(filters);\n const unordered = sort === undefined;\n // PK comparator is used for source-level overlay matching (remove by PK\n // equality) even when no ordering is requested.\n const primaryKeySort: Ordering = this.#primaryKey.map(k => [k, 'asc']);\n\n const input: SourceInput = {\n getSchema: () => schema,\n fetch: req => this.#fetch(req, connection),\n setOutput: output => {\n connection.output = output;\n },\n destroy: () => {\n const idx = this.#connections.indexOf(connection);\n assert(idx !== -1, 'Connection not found');\n this.#connections.splice(idx, 1);\n },\n fullyAppliedFilters: !transformedFilters.conditionsRemoved,\n };\n\n const connection: Connection = {\n input,\n debug,\n output: undefined,\n sort,\n splitEditKeys,\n filters: transformedFilters.filters\n ? {\n condition: transformedFilters.filters,\n predicate: createPredicate(transformedFilters.filters),\n }\n : undefined,\n compareRows: sort ? makeComparator(sort) : makeComparator(primaryKeySort),\n lastPushedEpoch: 0,\n };\n const schema = this.#getSchema(connection, unordered);\n if (!unordered) {\n assertOrderingIncludesPK(sort, this.#primaryKey);\n }\n\n this.#connections.push(connection);\n return input;\n }\n\n toSQLiteRow(row: Row): Row {\n return Object.fromEntries(\n Object.entries(row).map(([key, value]) => [\n key,\n toSQLiteType(value, this.#columns[key].type),\n ]),\n ) as Row;\n }\n\n #fetch(req: FetchRequest, connection: Connection): Stream<Node \| 'yield'> {\n const {sort, debug} = connection;\n\n const query = this.#requestToSQL(req, connection.filters?.condition, sort);\n const sqlAndBindings = format(query);\n\n const cachedStatement = this.#stmts.cache.get(sqlAndBindings.text);\n cachedStatement.statement.safeIntegers(true);\n const rowIterator = cachedStatement.statement.iterate<Row>(\n ...sqlAndBindings.values,\n );\n try {\n debug?.initQuery(this.#table, sqlAndBindings.text);\n\n if (sort) {\n const comparator = makeComparator(sort, req.reverse);\n yield generateWithStart(\n generateWithYields(\n generateWithOverlay(\n req.start?.row,\n this.#mapFromSQLiteTypes(\n this.#columns,\n rowIterator,\n sqlAndBindings.text,\n debug,\n ),\n req.constraint,\n this.#overlay,\n connection.lastPushedEpoch,\n comparator,\n connection.filters?.predicate,\n req.multiConstraints,\n ),\n this.#shouldYield,\n ),\n req.start,\n comparator,\n );\n } else {\n yield* generateWithYields(\n generateWithOverlayUnordered(\n this.#mapFromSQLiteTypes(\n this.#columns,\n rowIterator,\n sqlAndBindings.text,\n debug,\n ),\n req.constraint,\n this.#overlay,\n connection.lastPushedEpoch,\n this.#primaryKey,\n connection.filters?.predicate,\n req.multiConstraints,\n ),\n this.#shouldYield,\n );\n }\n } finally {\n // Ensure the SQLite iterate() is closed.\n rowIterator.return?.();\n if (debug) {\n let totalNvisit = 0;\n const planLines: string[] = [];\n for (let i = 0; ; i++) {\n const nvisit = cachedStatement.statement.scanStatus(\n i,\n SQLite3Database.SQLITE_SCANSTAT_NVISIT,\n 1,\n );\n if (nvisit === undefined) {\n break;\n }\n totalNvisit += Number(nvisit);\n const explain = cachedStatement.statement.scanStatus(\n i,\n SQLite3Database.SQLITE_SCANSTAT_EXPLAIN,\n 1,\n );\n if (typeof explain === 'string' && explain.length > 0) {\n planLines.push(explain);\n }\n }\n if (totalNvisit !== 0) {\n debug.recordNVisit(this.#table, sqlAndBindings.text, totalNvisit);\n }\n if (planLines.length > 0) {\n debug.recordExplain(this.#table, sqlAndBindings.text, planLines);\n }\n cachedStatement.statement.scanStatusReset();\n }\n this.#stmts.cache.return(cachedStatement);\n }\n }\n\n #mapFromSQLiteTypes(\n valueTypes: Record<string, SchemaValue>,\n rowIterator: IterableIterator<Row>,\n query: string,\n debug: DebugDelegate \| undefined,\n ): IterableIterator<Row> {\n let result;\n do {\n result = timeSampled(\n this.#lc,\n ++eventCount,\n this.#logConfig.ivmSampling,\n () => rowIterator.next(),\n this.#logConfig.slowRowThreshold,\n () =>\n `table-source.next took too long for ${query}. Are you missing an index?`,\n );\n if (result.done) {\n break;\n }\n const row = fromSQLiteTypes(valueTypes, result.value, this.#table);\n debug?.rowVended(this.#table, query, row);\n yield row;\n } while (!result.done);\n }\n\n push(change: SourceChange): Stream<'yield'> {\n for (const result of this.genPush(change)) {\n if (result === 'yield') {\n yield result;\n }\n }\n }\n\n genPush(change: SourceChange) {\n const exists = (row: Row) =>\n this.#stmts.checkExists.get<{exists: number} \| undefined>(\n ...toSQLiteTypes(this.#primaryKey, row, this.#columns),\n )?.exists === 1;\n const setOverlay = (o: Overlay \| undefined) => (this.#overlay = o);\n const writeChange = (c: SourceChange) => this.#writeChange(c);\n\n yield genPushAndWriteWithSplitEdit(\n this.#connections,\n change,\n exists,\n setOverlay,\n writeChange,\n () => ++this.#pushEpoch,\n );\n }\n\n #writeChange(change: SourceChange) {\n switch (change[SourceChangeIndex.TYPE]) {\n case ChangeType.ADD:\n this.#stmts.insert.run(\n ...toSQLiteTypes(\n // TODO(arv): Compute this once!\n Object.keys(this.#columns),\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n break;\n case ChangeType.REMOVE:\n this.#stmts.delete.run(\n ...toSQLiteTypes(\n this.#primaryKey,\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n break;\n case ChangeType.EDIT: {\n // If the PK is the same, use UPDATE.\n if (\n canUseUpdate(\n change[SourceChangeIndex.OLD_ROW],\n change[SourceChangeIndex.ROW],\n this.#columns,\n this.#primaryKey,\n )\n ) {\n const mergedRow = {\n ...change[SourceChangeIndex.OLD_ROW],\n ...change[SourceChangeIndex.ROW],\n };\n const params = [\n ...nonPrimaryValues(this.#columns, this.#primaryKey, mergedRow),\n ...toSQLiteTypes(this.#primaryKey, mergedRow, this.#columns),\n ];\n must(this.#stmts.update).run(params);\n } else {\n this.#stmts.delete.run(\n ...toSQLiteTypes(\n this.#primaryKey,\n change[SourceChangeIndex.OLD_ROW],\n this.#columns,\n ),\n );\n this.#stmts.insert.run(\n ...toSQLiteTypes(\n Object.keys(this.#columns),\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n }\n\n break;\n }\n default:\n unreachable(change);\n }\n }\n\n #getRowStmtCache = new Map<string, string>();\n\n #getRowStmt(keyCols: string[]): string {\n const keyString = JSON.stringify(keyCols);\n let stmt = this.#getRowStmtCache.get(keyString);\n if (!stmt) {\n stmt = compile(\n sql`SELECT ${this.#allColumns} FROM ${sql.ident(\n this.#table,\n )} WHERE ${sql.join(\n keyCols.map(k => sql`${sql.ident(k)}=?`),\n sql` AND`,\n )}`,\n );\n this.#getRowStmtCache.set(keyString, stmt);\n }\n return stmt;\n }\n\n /*\n Retrieves a row from the backing DB by a unique key, or `undefined` if such a\n * row does not exist. This is not used in the IVM pipeline but is useful\n * for retrieving data that is consistent with the state (and type\n * semantics) of the pipeline. Note that this key may not necessarily correspond\n * to the `primaryKey` with which this TableSource.\n /\n getRow(rowKey: Row): Row \| undefined {\n const keyCols = Object.keys(rowKey);\n\n const stmt = this.#getRowStmt(keyCols);\n const row = this.#stmts.cache.use(stmt, cached =>\n cached.statement\n .safeIntegers(true)\n .get<Row>(...toSQLiteTypes(keyCols, rowKey, this.#columns)),\n );\n if (row) {\n return fromSQLiteTypes(this.#columns, row, this.#table);\n }\n return row;\n }\n\n #requestToSQL(\n request: FetchRequest,\n filters: NoSubqueryCondition \| undefined,\n order: Ordering \| undefined,\n ): SQLQuery {\n return buildSelectQuery(\n this.#table,\n this.#columns,\n request.constraint,\n filters,\n order,\n request.reverse,\n request.start,\n request.multiConstraints,\n );\n }\n}\n\nfunction getUniqueIndexes(\n db: Database,\n tableName: string,\n): Map<string, Set<string>> {\n const sqlAndBindings = format(\n sql`\n SELECT idx.name, json_group_array(col.name) as columnsJSON\n FROM sqlite_master as idx\n JOIN pragma_index_list(idx.tbl_name) AS info ON info.name = idx.name\n JOIN pragma_index_info(idx.name) as col\n WHERE idx.tbl_name = ${tableName} AND\n idx.type = 'index' AND \n info.\"unique\" != 0\n GROUP BY idx.name\n ORDER BY idx.name`,\n );\n const stmt = db.prepare(sqlAndBindings.text);\n const indexes = stmt.all<{columnsJSON: string}>(...sqlAndBindings.values);\n return new Map(\n indexes.map(({columnsJSON}) => {\n const columns = JSON.parse(columnsJSON);\n const set = new Set<string>(columns);\n return [JSON.stringify(columns.sort()), set];\n }),\n );\n}\n\nexport function toSQLiteTypes(\n columns: readonly string[],\n row: Row,\n columnTypes: Record<string, SchemaValue>,\n): readonly unknown[] {\n return columns.map(col => toSQLiteType(row[col], columnTypes[col].type));\n}\n\nexport function toSQLiteTypeName(type: ValueType) {\n switch (type) {\n case 'boolean':\n return 'INTEGER';\n case 'number':\n return 'REAL';\n case 'string':\n return 'TEXT';\n case 'null':\n return 'NULL';\n case 'json':\n return 'TEXT';\n }\n}\n\nexport function fromSQLiteTypes(\n valueTypes: Record<string, SchemaValue>,\n row: Row,\n tableName: string,\n): Row {\n const newRow: Writable<Row> = {};\n for (const key of Object.keys(row)) {\n const valueType = valueTypes[key];\n if (valueType === undefined) {\n const columnList = Object.keys(valueTypes).sort().join(', ');\n throw new Error(\n `Invalid column \"${key}\" for table \"${tableName}\". Synced columns include ${columnList}`,\n );\n }\n newRow[key] = fromSQLiteType(valueType.type, row[key], key, tableName);\n }\n return newRow;\n}\n\nfunction fromSQLiteType(\n valueType: ValueType,\n v: Value,\n column: string,\n tableName: string,\n): Value {\n if (v === null) {\n return null;\n }\n switch (valueType) {\n case 'boolean':\n return !!v;\n case 'number':\n case 'string':\n case 'null':\n if (typeof v === 'bigint') {\n if (v > Number.MAX_SAFE_INTEGER \|\| v < Number.MIN_SAFE_INTEGER) {\n throw new UnsupportedValueError(\n `value ${v} (in ${tableName}.${column}) is outside of supported bounds`,\n );\n }\n return Number(v);\n }\n return v;\n case 'json':\n try {\n return JSON.parse(v as string);\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n throw new UnsupportedValueError(\n `Failed to parse JSON for ${tableName}.${column}: ${errorMessage}`,\n {cause: error},\n );\n }\n }\n}\n\nexport class UnsupportedValueError extends Error {}\n\nfunction canUseUpdate(\n oldRow: Row,\n row: Row,\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n): boolean {\n for (const pk of primaryKey) {\n if (oldRow[pk] !== row[pk]) {\n return false;\n }\n }\n return Object.keys(columns).length > primaryKey.length;\n}\n\nfunction nonPrimaryValues(\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n row: Row,\n): Iterable<unknown> {\n return nonPrimaryKeys(columns, primaryKey).map(c =>\n toSQLiteType(row[c], columns[c].type),\n );\n}\n\nfunction nonPrimaryKeys(\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n) {\n return Object.keys(columns).filter(c => !primaryKey.includes(c));\n}\n\nfunction generateWithYields(stream: Stream<Node>, shouldYield: () => boolean) {\n for (const n of stream) {\n if (shouldYield()) {\n yield 'yield';\n }\n yield n;\n }\n}\n"],"mappings":";;;;;;;;;;;;AA0DA,IAAI,aAAa;;;;;;;;;;;;;;;AAgBjB,IAAa,cAAb,MAA2C;CACzC,2BAAoB,IAAI,QAA8B;CACtD,eAAsC,CAAC;CACvC;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;;;;;;;CAQb,YACE,YACA,WACA,IACA,WACA,SACA,YACA,oBAAoB,OACpB;EACA,KAAKO,MAAM;EACX,KAAKD,aAAa;EAClB,KAAKJ,SAAS;EACd,KAAKC,WAAW;EAChB,KAAKC,iBAAiB,iBAAiB,IAAI,SAAS;EACpD,KAAKC,cAAc;EACnB,KAAKI,SAAS,KAAKC,kBAAkB,EAAE;EACvC,KAAKF,eAAe;EAEpB,OACE,KAAKJ,eAAe,IAAI,KAAK,UAAU,WAAW,SAAS,CAAC,CAAC,GAC7D,eAAe,WAAW,8BAC5B;CACF;CAEA,IAAI,cAAc;EAChB,OAAO;GACL,MAAM,KAAKF;GACX,SAAS,KAAKC;GACd,YAAY,KAAKE;EACnB;CACF;;;;;CAMA,MAAM,IAAc;EAClB,KAAKI,SAAS,KAAKC,kBAAkB,EAAE;CACzC;CAEA,kBAAkB,IAAc;EAC9B,MAAM,SAAS,KAAKV,SAAS,IAAI,EAAE;EACnC,IAAI,QACF,OAAO;EAGT,MAAM,QAAQ;GACZ,OAAO,IAAI,eAAe,EAAE;GAC5B,QAAQ,GAAG,QACT,QACE,GAAG,eAAe,IAAI,MAAM,KAAKE,MAAM,EAAE,IAAI,IAAI,KAC/C,OAAO,KAAK,KAAKC,QAAQ,EAAE,KAAI,MAAK,IAAI,MAAM,CAAC,CAAC,GAChD,IACF,EAAE,YAAY,IAAI,sBAChB,MAAM,KAAK,EAAC,QAAQ,OAAO,KAAK,KAAKA,QAAQ,EAAE,OAAM,CAAC,EACnD,KAAK,GAAG,EACR,KAAK,GAAG,CACb,EAAE,EACJ,CACF;GACA,QAAQ,GAAG,QACT,QACE,GAAG,eAAe,IAAI,MAAM,KAAKD,MAAM,EAAE,SAAS,IAAI,KACpD,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF;GAEA,QACE,OAAO,KAAK,KAAKF,QAAQ,EAAE,SAAS,KAAKE,YAAY,SACjD,GAAG,QACD,QACE,GAAG,UAAU,IAAI,MAAM,KAAKH,MAAM,EAAE,OAAO,IAAI,KAC7C,eAAe,KAAKC,UAAU,KAAKE,WAAW,EAAE,KAC9C,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAC1B,GACA,GACF,EAAE,SAAS,IAAI,KACb,KAAKA,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF,IACA,KAAA;GACN,aAAa,GAAG,QACd,QACE,GAAG,6BAA6B,IAAI,MAClC,KAAKH,MACP,EAAE,SAAS,IAAI,KACb,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,EAAE,SACJ,CACF;GACA,aAAa,GAAG,QACd,QACE,GAAG,iBAAiB,IAAI,MAAM,KAAKH,MAAM,EAAE,SAAS,IAAI,KACtD,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF;EACF;EACA,KAAKL,SAAS,IAAI,IAAI,KAAK;EAC3B,OAAO;CACT;CAEA,IAAIW,cAAc;EAChB,OAAO,IAAI,KACT,OAAO,KAAK,KAAKR,QAAQ,EAAE,KAAI,MAAK,IAAI,MAAM,CAAC,CAAC,GAChD,GAAG,GACL;CACF;CAEA,WAAW,YAAwB,WAAkC;EACnE,OAAO;GACL,WAAW,KAAKD;GAChB,SAAS,KAAKC;GACd,YAAY,KAAKE;GACjB,MAAM,YAAY,KAAA,IAAY,WAAW;GACzC,eAAe,CAAC;GAChB,UAAU;GACV,QAAQ;GACR,aAAa,WAAW;EAC1B;CACF;CAEA,QACE,MACA,SACA,eACA,OACA;EACA,MAAM,qBAAqB,iBAAiB,OAAO;EACnD,MAAM,YAAY,SAAS,KAAA;EAG3B,MAAM,iBAA2B,KAAKA,YAAY,KAAI,MAAK,CAAC,GAAG,KAAK,CAAC;EAErE,MAAM,QAAqB;GACzB,iBAAiB;GACjB,QAAO,QAAO,KAAKO,OAAO,KAAK,UAAU;GACzC,YAAW,WAAU;IACnB,WAAW,SAAS;GACtB;GACA,eAAe;IACb,MAAM,MAAM,KAAKX,aAAa,QAAQ,UAAU;IAChD,OAAO,QAAQ,IAAI,sBAAsB;IACzC,KAAKA,aAAa,OAAO,KAAK,CAAC;GACjC;GACA,qBAAqB,CAAC,mBAAmB;EAC3C;EAEA,MAAM,aAAyB;GAC7B;GACA;GACA,QAAQ,KAAA;GACR;GACA;GACA,SAAS,mBAAmB,UACxB;IACE,WAAW,mBAAmB;IAC9B,WAAW,gBAAgB,mBAAmB,OAAO;GACvD,IACA,KAAA;GACJ,aAAa,OAAO,eAAe,IAAI,IAAI,eAAe,cAAc;GACxE,iBAAiB;EACnB;EACA,MAAM,SAAS,KAAKY,WAAW,YAAY,SAAS;EACpD,IAAI,CAAC,WACH,yBAAyB,MAAM,KAAKR,WAAW;EAGjD,KAAKJ,aAAa,KAAK,UAAU;EACjC,OAAO;CACT;CAEA,YAAY,KAAe;EACzB,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,EAAE,KAAK,CAAC,KAAK,WAAW,CACxC,KACA,aAAa,OAAO,KAAKE,SAAS,KAAK,IAAI,CAC7C,CAAC,CACH;CACF;CAEA,CAACS,OAAO,KAAmB,YAAgD;EACzE,MAAM,EAAC,MAAM,UAAS;EAGtB,MAAM,iBAAiB,OADT,KAAKE,cAAc,KAAK,WAAW,SAAS,WAAW,IACvC,CAAK;EAEnC,MAAM,kBAAkB,KAAKL,OAAO,MAAM,IAAI,eAAe,IAAI;EACjE,gBAAgB,UAAU,aAAa,IAAI;EAC3C,MAAM,cAAc,gBAAgB,UAAU,QAC5C,GAAG,eAAe,MACpB;EACA,IAAI;GACF,OAAO,UAAU,KAAKP,QAAQ,eAAe,IAAI;GAEjD,IAAI,MAAM;IACR,MAAM,aAAa,eAAe,MAAM,IAAI,OAAO;IACnD,OAAO,kBACL,mBACE,oBACE,IAAI,OAAO,KACX,KAAKa,oBACH,KAAKZ,UACL,aACA,eAAe,MACf,KACF,GACA,IAAI,YACJ,KAAKa,UACL,WAAW,iBACX,YACA,WAAW,SAAS,WACpB,IAAI,gBACN,GACA,KAAKR,YACP,GACA,IAAI,OACJ,UACF;GACF,OACE,OAAO,mBACL,6BACE,KAAKO,oBACH,KAAKZ,UACL,aACA,eAAe,MACf,KACF,GACA,IAAI,YACJ,KAAKa,UACL,WAAW,iBACX,KAAKX,aACL,WAAW,SAAS,WACpB,IAAI,gBACN,GACA,KAAKG,YACP;EAEJ,UAAU;GAER,YAAY,SAAS;GACrB,IAAI,OAAO;IACT,IAAI,cAAc;IAClB,MAAM,YAAsB,CAAC;IAC7B,KAAK,IAAI,IAAI,IAAK,KAAK;KACrB,MAAM,SAAS,gBAAgB,UAAU,WACvC,GACA,gBAAgB,wBAChB,CACF;KACA,IAAI,WAAW,KAAA,GACb;KAEF,eAAe,OAAO,MAAM;KAC5B,MAAM,UAAU,gBAAgB,UAAU,WACxC,GACA,gBAAgB,yBAChB,CACF;KACA,IAAI,OAAO,YAAY,YAAY,QAAQ,SAAS,GAClD,UAAU,KAAK,OAAO;IAE1B;IACA,IAAI,gBAAgB,GAClB,MAAM,aAAa,KAAKN,QAAQ,eAAe,MAAM,WAAW;IAElE,IAAI,UAAU,SAAS,GACrB,MAAM,cAAc,KAAKA,QAAQ,eAAe,MAAM,SAAS;IAEjE,gBAAgB,UAAU,gBAAgB;GAC5C;GACA,KAAKO,OAAO,MAAM,OAAO,eAAe;EAC1C;CACF;CAEA,CAACM,oBACC,YACA,aACA,OACA,OACuB;EACvB,IAAI;EACJ,GAAG;GACD,SAAS,YACP,KAAKR,KACL,EAAE,YACF,KAAKD,WAAW,mBACV,YAAY,KAAK,GACvB,KAAKA,WAAW,wBAEd,uCAAuC,MAAM,4BACjD;GACA,IAAI,OAAO,MACT;GAEF,MAAM,MAAM,gBAAgB,YAAY,OAAO,OAAO,KAAKJ,MAAM;GACjE,OAAO,UAAU,KAAKA,QAAQ,OAAO,GAAG;GACxC,MAAM;EACR,SAAS,CAAC,OAAO;CACnB;CAEA,CAAC,KAAK,QAAuC;EAC3C,KAAK,MAAM,UAAU,KAAK,QAAQ,MAAM,GACtC,IAAI,WAAW,SACb,MAAM;CAGZ;CAEA,CAAC,QAAQ,QAAsB;EAC7B,MAAM,UAAU,QACd,KAAKO,OAAO,YAAY,IACtB,GAAG,cAAc,KAAKJ,aAAa,KAAK,KAAKF,QAAQ,CACvD,GAAG,WAAW;EAChB,MAAM,cAAc,MAA4B,KAAKa,WAAW;EAChE,MAAM,eAAe,MAAoB,KAAKC,aAAa,CAAC;EAE5D,OAAO,6BACL,KAAKhB,cACL,QACA,QACA,YACA,mBACM,EAAE,KAAKiB,UACf;CACF;CAEA,aAAa,QAAsB;EACjC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAKT,OAAO,OAAO,IACjB,GAAG,cAED,OAAO,KAAK,KAAKN,QAAQ,GACzB,OAAO,IACP,KAAKA,QACP,CACF;IACA;GACF,KAAK;IACH,KAAKM,OAAO,OAAO,IACjB,GAAG,cACD,KAAKJ,aACL,OAAO,IACP,KAAKF,QACP,CACF;IACA;GACF,KAAK;IAEH,IACE,aACE,OAAO,IACP,OAAO,IACP,KAAKA,UACL,KAAKE,WACP,GACA;KACA,MAAM,YAAY;MAChB,GAAG,OAAO;MACV,GAAG,OAAO;KACZ;KACA,MAAM,SAAS,CACb,GAAG,iBAAiB,KAAKF,UAAU,KAAKE,aAAa,SAAS,GAC9D,GAAG,cAAc,KAAKA,aAAa,WAAW,KAAKF,QAAQ,CAC7D;KACA,KAAK,KAAKM,OAAO,MAAM,EAAE,IAAI,MAAM;IACrC,OAAO;KACL,KAAKA,OAAO,OAAO,IACjB,GAAG,cACD,KAAKJ,aACL,OAAO,IACP,KAAKF,QACP,CACF;KACA,KAAKM,OAAO,OAAO,IACjB,GAAG,cACD,OAAO,KAAK,KAAKN,QAAQ,GACzB,OAAO,IACP,KAAKA,QACP,CACF;IACF;IAEA;GAEF,SACE,YAAY,MAAM;EACtB;CACF;CAEA,mCAAmB,IAAI,IAAoB;CAE3C,YAAY,SAA2B;EACrC,MAAM,YAAY,KAAK,UAAU,OAAO;EACxC,IAAI,OAAO,KAAKgB,iBAAiB,IAAI,SAAS;EAC9C,IAAI,CAAC,MAAM;GACT,OAAO,QACL,GAAG,UAAU,KAAKR,YAAY,QAAQ,IAAI,MACxC,KAAKT,MACP,EAAE,SAAS,IAAI,KACb,QAAQ,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GACvC,GAAG,MACL,GACF;GACA,KAAKiB,iBAAiB,IAAI,WAAW,IAAI;EAC3C;EACA,OAAO;CACT;;;;;;;;CASA,OAAO,QAA8B;EACnC,MAAM,UAAU,OAAO,KAAK,MAAM;EAElC,MAAM,OAAO,KAAKC,YAAY,OAAO;EACrC,MAAM,MAAM,KAAKX,OAAO,MAAM,IAAI,OAAM,WACtC,OAAO,UACJ,aAAa,IAAI,EACjB,IAAS,GAAG,cAAc,SAAS,QAAQ,KAAKN,QAAQ,CAAC,CAC9D;EACA,IAAI,KACF,OAAO,gBAAgB,KAAKA,UAAU,KAAK,KAAKD,MAAM;EAExD,OAAO;CACT;CAEA,cACE,SACA,SACA,OACU;EACV,OAAO,iBACL,KAAKA,QACL,KAAKC,UACL,QAAQ,YACR,SACA,OACA,QAAQ,SACR,QAAQ,OACR,QAAQ,gBACV;CACF;AACF;AAEA,SAAS,iBACP,IACA,WAC0B;CAC1B,MAAM,iBAAiB,OACrB,GAAG;;;;;6BAKsB,UAAU;;;;wBAKrC;CAEA,MAAM,UADO,GAAG,QAAQ,eAAe,IACvB,EAAK,IAA2B,GAAG,eAAe,MAAM;CACxE,OAAO,IAAI,IACT,QAAQ,KAAK,EAAC,kBAAiB;EAC7B,MAAM,UAAU,KAAK,MAAM,WAAW;EACtC,MAAM,MAAM,IAAI,IAAY,OAAO;EACnC,OAAO,CAAC,KAAK,UAAU,QAAQ,KAAK,CAAC,GAAG,GAAG;CAC7C,CAAC,CACH;AACF;AAEA,SAAgB,cACd,SACA,KACA,aACoB;CACpB,OAAO,QAAQ,KAAI,QAAO,aAAa,IAAI,MAAM,YAAY,KAAK,IAAI,CAAC;AACzE;AAiBA,SAAgB,gBACd,YACA,KACA,WACK;CACL,MAAM,SAAwB,CAAC;CAC/B,KAAK,MAAM,OAAO,OAAO,KAAK,GAAG,GAAG;EAClC,MAAM,YAAY,WAAW;EAC7B,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,aAAa,OAAO,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,IAAI;GAC3D,MAAM,IAAI,MACR,mBAAmB,IAAI,eAAe,UAAU,4BAA4B,YAC9E;EACF;EACA,OAAO,OAAO,eAAe,UAAU,MAAM,IAAI,MAAM,KAAK,SAAS;CACvE;CACA,OAAO;AACT;AAEA,SAAS,eACP,WACA,GACA,QACA,WACO;CACP,IAAI,MAAM,MACR,OAAO;CAET,QAAQ,WAAR;EACE,KAAK,WACH,OAAO,CAAC,CAAC;EACX,KAAK;EACL,KAAK;EACL,KAAK;GACH,IAAI,OAAO,MAAM,UAAU;IACzB,IAAI,IAAI,OAAO,oBAAoB,IAAI,OAAO,kBAC5C,MAAM,IAAI,sBACR,SAAS,EAAE,OAAO,UAAU,GAAG,OAAO,iCACxC;IAEF,OAAO,OAAO,CAAC;GACjB;GACA,OAAO;EACT,KAAK,QACH,IAAI;GACF,OAAO,KAAK,MAAM,CAAW;EAC/B,SAAS,OAAO;GAGd,MAAM,IAAI,sBACR,4BAA4B,UAAU,GAAG,OAAO,IAFhD,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAGrD,EAAC,OAAO,MAAK,CACf;EACF;CACJ;AACF;AAEA,IAAa,wBAAb,cAA2C,MAAM,CAAC;AAElD,SAAS,aACP,QACA,KACA,SACA,YACS;CACT,KAAK,MAAM,MAAM,YACf,IAAI,OAAO,QAAQ,IAAI,KACrB,OAAO;CAGX,OAAO,OAAO,KAAK,OAAO,EAAE,SAAS,WAAW;AAClD;AAEA,SAAS,iBACP,SACA,YACA,KACmB;CACnB,OAAO,eAAe,SAAS,UAAU,EAAE,KAAI,MAC7C,aAAa,IAAI,IAAI,QAAQ,GAAG,IAAI,CACtC;AACF;AAEA,SAAS,eACP,SACA,YACA;CACA,OAAO,OAAO,KAAK,OAAO,EAAE,QAAO,MAAK,CAAC,WAAW,SAAS,CAAC,CAAC;AACjE;AAEA,UAAU,mBAAmB,QAAsB,aAA4B;CAC7E,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,YAAY,GACd,MAAM;EAER,MAAM;CACR;AACF"}
1	+ {"version":3,"file":"table-source.js","names":["#dbCache","#connections","#table","#columns","#uniqueIndexes","#primaryKey","#logConfig","#lc","#shouldYield","#stmts","#getStatementsFor","#allColumns","#fetch","#getSchema","#requestToSQL","#mapFromSQLiteTypes","#overlay","#writeChange","#pushEpoch","#getRowStmtCache","#getRowStmt"],"sources":["../../../../zqlite/src/table-source.ts"],"sourcesContent":["import type {SQLQuery} from '@databases/sql';\nimport type {LogContext} from '@rocicorp/logger';\nimport SQLite3Database from '@rocicorp/zero-sqlite3';\nimport type {LogConfig} from '../../otel/src/log-options.ts';\nimport {timeSampled} from '../../otel/src/maybe-time.ts';\nimport {assert, unreachable} from '../../shared/src/asserts.ts';\nimport {must} from '../../shared/src/must.ts';\nimport type {Writable} from '../../shared/src/writable.ts';\nimport type {Condition, Ordering} from '../../zero-protocol/src/ast.ts';\nimport type {Row, Value} from '../../zero-protocol/src/data.ts';\nimport type {PrimaryKey} from '../../zero-protocol/src/primary-key.ts';\nimport type {\n SchemaValue,\n ValueType,\n} from '../../zero-schema/src/table-schema.ts';\nimport type {DebugDelegate} from '../../zql/src/builder/debug-delegate.ts';\nimport {\n createPredicate,\n transformFilters,\n} from '../../zql/src/builder/filter.ts';\nimport {ChangeType} from '../../zql/src/ivm/change-type.ts';\nimport {makeComparator, type Node} from '../../zql/src/ivm/data.ts';\nimport {\n generateWithOverlay,\n generateWithOverlayUnordered,\n generateWithStart,\n genPushAndWriteWithSplitEdit,\n type Connection,\n type Overlay,\n} from '../../zql/src/ivm/memory-source.ts';\nimport {type FetchRequest} from '../../zql/src/ivm/operator.ts';\nimport type {SourceSchema} from '../../zql/src/ivm/schema.ts';\nimport {SourceChangeIndex} from '../../zql/src/ivm/source-change-index.ts';\nimport {\n type Source,\n type SourceChange,\n type SourceInput,\n} from '../../zql/src/ivm/source.ts';\nimport type {Stream} from '../../zql/src/ivm/stream.ts';\nimport {assertOrderingIncludesPK} from '../../zql/src/query/complete-ordering.ts';\nimport type {Database, Statement} from './db.ts';\nimport {compile, format, sql} from './internal/sql.ts';\nimport {StatementCache} from './internal/statement-cache.ts';\nimport {\n buildSelectQuery,\n toSQLiteType,\n type NoSubqueryCondition,\n} from './query-builder.ts';\n\ntype Statements = {\n readonly cache: StatementCache;\n readonly insert: Statement;\n readonly delete: Statement;\n readonly update: Statement \| undefined;\n readonly checkExists: Statement;\n readonly getExisting: Statement;\n};\n\nlet eventCount = 0;\n\n/*\n A source that is backed by a SQLite table.\n \n Values are written to the backing table _after_ being vended by the source.\n \n This ordering of events is to ensure self joins function properly. That is,\n * we can't reveal a value to an output before it has been pushed to that output.\n \n The code is fairly straightforward except for:\n * 1. Dealing with a `fetch` that has a basis of `before`.\n * 2. Dealing with compound orders that have differing directions (a ASC, b DESC, c ASC)\n \n See comments in relevant functions for more details.\n /\nexport class TableSource implements Source {\n readonly #dbCache = new WeakMap<Database, Statements>();\n readonly #connections: Connection[] = [];\n readonly #table: string;\n readonly #columns: Record<string, SchemaValue>;\n // Maps sorted columns JSON string (e.g. '[\"a\",\"b\"]) to Set of columns.\n readonly #uniqueIndexes: Map<string, Set<string>>;\n readonly #primaryKey: PrimaryKey;\n readonly #logConfig: LogConfig;\n readonly #lc: LogContext;\n readonly #shouldYield: () => boolean;\n #stmts: Statements;\n #overlay?: Overlay \| undefined;\n #pushEpoch = 0;\n\n /\n @param shouldYield a function called after each row is read from the database,\n * which should return true if the source should yield the special 'yield' value\n * to yield control back to the caller at the end of the pipeline. Can\n * also throw an error to abort the pipeline processing.\n /\n constructor(\n logContext: LogContext,\n logConfig: LogConfig,\n db: Database,\n tableName: string,\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n shouldYield = () => false,\n ) {\n this.#lc = logContext;\n this.#logConfig = logConfig;\n this.#table = tableName;\n this.#columns = columns;\n this.#uniqueIndexes = getUniqueIndexes(db, tableName);\n this.#primaryKey = primaryKey;\n this.#stmts = this.#getStatementsFor(db);\n this.#shouldYield = shouldYield;\n\n assert(\n this.#uniqueIndexes.has(JSON.stringify(primaryKey.toSorted())),\n `primary key ${primaryKey} does not have a UNIQUE index`,\n );\n }\n\n get tableSchema() {\n return {\n name: this.#table,\n columns: this.#columns,\n primaryKey: this.#primaryKey,\n };\n }\n\n /\n Sets the db (snapshot) to use, to facilitate the Snapshotter leapfrog\n * algorithm for concurrent traversal of historic timelines.\n /\n setDB(db: Database) {\n this.#stmts = this.#getStatementsFor(db);\n }\n\n #getStatementsFor(db: Database) {\n const cached = this.#dbCache.get(db);\n if (cached) {\n return cached;\n }\n\n const stmts = {\n cache: new StatementCache(db),\n insert: db.prepare(\n compile(\n sql`INSERT INTO ${sql.ident(this.#table)} (${sql.join(\n Object.keys(this.#columns).map(c => sql.ident(c)),\n ', ',\n )}) VALUES (${sql.__dangerous__rawValue(\n Array.from({length: Object.keys(this.#columns).length})\n .fill('?')\n .join(','),\n )})`,\n ),\n ),\n delete: db.prepare(\n compile(\n sql`DELETE FROM ${sql.ident(this.#table)} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n ),\n // If all the columns are part of the primary key, we cannot use UPDATE.\n update:\n Object.keys(this.#columns).length > this.#primaryKey.length\n ? db.prepare(\n compile(\n sql`UPDATE ${sql.ident(this.#table)} SET ${sql.join(\n nonPrimaryKeys(this.#columns, this.#primaryKey).map(\n c => sql`${sql.ident(c)}=?`,\n ),\n ',',\n )} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n )\n : undefined,\n checkExists: db.prepare(\n compile(\n sql`SELECT 1 AS \"exists\" FROM ${sql.ident(\n this.#table,\n )} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )} LIMIT 1`,\n ),\n ),\n getExisting: db.prepare(\n compile(\n sql`SELECT FROM ${sql.ident(this.#table)} WHERE ${sql.join(\n this.#primaryKey.map(k => sql`${sql.ident(k)}=?`),\n ' AND ',\n )}`,\n ),\n ),\n };\n this.#dbCache.set(db, stmts);\n return stmts;\n }\n\n get #allColumns() {\n return sql.join(\n Object.keys(this.#columns).map(c => sql.ident(c)),\n sql`,`,\n );\n }\n\n #getSchema(connection: Connection, unordered: boolean): SourceSchema {\n return {\n tableName: this.#table,\n columns: this.#columns,\n primaryKey: this.#primaryKey,\n sort: unordered ? undefined : connection.sort,\n relationships: {},\n isHidden: false,\n system: 'client',\n compareRows: connection.compareRows,\n };\n }\n\n connect(\n sort: Ordering \| undefined,\n filters?: Condition,\n splitEditKeys?: Set<string>,\n debug?: DebugDelegate,\n ) {\n const transformedFilters = transformFilters(filters);\n const unordered = sort === undefined;\n // PK comparator is used for source-level overlay matching (remove by PK\n // equality) even when no ordering is requested.\n const primaryKeySort: Ordering = this.#primaryKey.map(k => [k, 'asc']);\n\n const input: SourceInput = {\n getSchema: () => schema,\n fetch: req => this.#fetch(req, connection),\n setOutput: output => {\n connection.output = output;\n },\n destroy: () => {\n const idx = this.#connections.indexOf(connection);\n assert(idx !== -1, 'Connection not found');\n this.#connections.splice(idx, 1);\n },\n fullyAppliedFilters: !transformedFilters.conditionsRemoved,\n };\n\n const connection: Connection = {\n input,\n debug,\n output: undefined,\n sort,\n splitEditKeys,\n filters: transformedFilters.filters\n ? {\n condition: transformedFilters.filters,\n predicate: createPredicate(transformedFilters.filters),\n }\n : undefined,\n compareRows: sort ? makeComparator(sort) : makeComparator(primaryKeySort),\n lastPushedEpoch: 0,\n };\n const schema = this.#getSchema(connection, unordered);\n if (!unordered) {\n assertOrderingIncludesPK(sort, this.#primaryKey);\n }\n\n this.#connections.push(connection);\n return input;\n }\n\n toSQLiteRow(row: Row): Row {\n return Object.fromEntries(\n Object.entries(row).map(([key, value]) => [\n key,\n toSQLiteType(value, this.#columns[key].type),\n ]),\n ) as Row;\n }\n\n #fetch(req: FetchRequest, connection: Connection): Stream<Node \| 'yield'> {\n const {sort, debug} = connection;\n\n const query = this.#requestToSQL(req, connection.filters?.condition, sort);\n const sqlAndBindings = format(query);\n\n const cachedStatement = this.#stmts.cache.get(sqlAndBindings.text);\n cachedStatement.statement.safeIntegers(true);\n const rowIterator = cachedStatement.statement.iterate<Row>(\n ...sqlAndBindings.values,\n );\n try {\n debug?.initQuery(this.#table, sqlAndBindings.text);\n\n if (sort) {\n const comparator = makeComparator(sort, req.reverse);\n yield generateWithStart(\n generateWithYields(\n generateWithOverlay(\n req.start?.row,\n this.#mapFromSQLiteTypes(\n this.#columns,\n rowIterator,\n sqlAndBindings.text,\n debug,\n ),\n req.constraint,\n this.#overlay,\n connection.lastPushedEpoch,\n comparator,\n // SQL does the ordering and constraining, so the row stream is\n // already in the connection's sort order: the splice comparator\n // and the `startAt` comparator coincide here.\n comparator,\n connection.filters?.predicate,\n req.multiConstraints,\n ),\n this.#shouldYield,\n ),\n req.start,\n comparator,\n );\n } else {\n yield* generateWithYields(\n generateWithOverlayUnordered(\n this.#mapFromSQLiteTypes(\n this.#columns,\n rowIterator,\n sqlAndBindings.text,\n debug,\n ),\n req.constraint,\n this.#overlay,\n connection.lastPushedEpoch,\n this.#primaryKey,\n connection.filters?.predicate,\n req.multiConstraints,\n ),\n this.#shouldYield,\n );\n }\n } finally {\n // Ensure the SQLite iterate() is closed.\n rowIterator.return?.();\n if (debug) {\n let totalNvisit = 0;\n const planLines: string[] = [];\n for (let i = 0; ; i++) {\n const nvisit = cachedStatement.statement.scanStatus(\n i,\n SQLite3Database.SQLITE_SCANSTAT_NVISIT,\n 1,\n );\n if (nvisit === undefined) {\n break;\n }\n totalNvisit += Number(nvisit);\n const explain = cachedStatement.statement.scanStatus(\n i,\n SQLite3Database.SQLITE_SCANSTAT_EXPLAIN,\n 1,\n );\n if (typeof explain === 'string' && explain.length > 0) {\n planLines.push(explain);\n }\n }\n if (totalNvisit !== 0) {\n debug.recordNVisit(this.#table, sqlAndBindings.text, totalNvisit);\n }\n if (planLines.length > 0) {\n debug.recordExplain(this.#table, sqlAndBindings.text, planLines);\n }\n cachedStatement.statement.scanStatusReset();\n }\n this.#stmts.cache.return(cachedStatement);\n }\n }\n\n #mapFromSQLiteTypes(\n valueTypes: Record<string, SchemaValue>,\n rowIterator: IterableIterator<Row>,\n query: string,\n debug: DebugDelegate \| undefined,\n ): IterableIterator<Row> {\n let result;\n do {\n result = timeSampled(\n this.#lc,\n ++eventCount,\n this.#logConfig.ivmSampling,\n () => rowIterator.next(),\n this.#logConfig.slowRowThreshold,\n () =>\n `table-source.next took too long for ${query}. Are you missing an index?`,\n );\n if (result.done) {\n break;\n }\n const row = fromSQLiteTypes(valueTypes, result.value, this.#table);\n debug?.rowVended(this.#table, query, row);\n yield row;\n } while (!result.done);\n }\n\n push(change: SourceChange): Stream<'yield'> {\n for (const result of this.genPush(change)) {\n if (result === 'yield') {\n yield result;\n }\n }\n }\n\n genPush(change: SourceChange) {\n const exists = (row: Row) =>\n this.#stmts.checkExists.get<{exists: number} \| undefined>(\n ...toSQLiteTypes(this.#primaryKey, row, this.#columns),\n )?.exists === 1;\n const setOverlay = (o: Overlay \| undefined) => (this.#overlay = o);\n const writeChange = (c: SourceChange) => this.#writeChange(c);\n\n yield genPushAndWriteWithSplitEdit(\n this.#connections,\n change,\n exists,\n setOverlay,\n writeChange,\n () => ++this.#pushEpoch,\n );\n }\n\n #writeChange(change: SourceChange) {\n switch (change[SourceChangeIndex.TYPE]) {\n case ChangeType.ADD:\n this.#stmts.insert.run(\n ...toSQLiteTypes(\n // TODO(arv): Compute this once!\n Object.keys(this.#columns),\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n break;\n case ChangeType.REMOVE:\n this.#stmts.delete.run(\n ...toSQLiteTypes(\n this.#primaryKey,\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n break;\n case ChangeType.EDIT: {\n // If the PK is the same, use UPDATE.\n if (\n canUseUpdate(\n change[SourceChangeIndex.OLD_ROW],\n change[SourceChangeIndex.ROW],\n this.#columns,\n this.#primaryKey,\n )\n ) {\n const mergedRow = {\n ...change[SourceChangeIndex.OLD_ROW],\n ...change[SourceChangeIndex.ROW],\n };\n const params = [\n ...nonPrimaryValues(this.#columns, this.#primaryKey, mergedRow),\n ...toSQLiteTypes(this.#primaryKey, mergedRow, this.#columns),\n ];\n must(this.#stmts.update).run(params);\n } else {\n this.#stmts.delete.run(\n ...toSQLiteTypes(\n this.#primaryKey,\n change[SourceChangeIndex.OLD_ROW],\n this.#columns,\n ),\n );\n this.#stmts.insert.run(\n ...toSQLiteTypes(\n Object.keys(this.#columns),\n change[SourceChangeIndex.ROW],\n this.#columns,\n ),\n );\n }\n\n break;\n }\n default:\n unreachable(change);\n }\n }\n\n #getRowStmtCache = new Map<string, string>();\n\n #getRowStmt(keyCols: string[]): string {\n const keyString = JSON.stringify(keyCols);\n let stmt = this.#getRowStmtCache.get(keyString);\n if (!stmt) {\n stmt = compile(\n sql`SELECT ${this.#allColumns} FROM ${sql.ident(\n this.#table,\n )} WHERE ${sql.join(\n keyCols.map(k => sql`${sql.ident(k)}=?`),\n sql` AND`,\n )}`,\n );\n this.#getRowStmtCache.set(keyString, stmt);\n }\n return stmt;\n }\n\n /*\n Retrieves a row from the backing DB by a unique key, or `undefined` if such a\n * row does not exist. This is not used in the IVM pipeline but is useful\n * for retrieving data that is consistent with the state (and type\n * semantics) of the pipeline. Note that this key may not necessarily correspond\n * to the `primaryKey` with which this TableSource.\n /\n getRow(rowKey: Row): Row \| undefined {\n const keyCols = Object.keys(rowKey);\n\n const stmt = this.#getRowStmt(keyCols);\n const row = this.#stmts.cache.use(stmt, cached =>\n cached.statement\n .safeIntegers(true)\n .get<Row>(...toSQLiteTypes(keyCols, rowKey, this.#columns)),\n );\n if (row) {\n return fromSQLiteTypes(this.#columns, row, this.#table);\n }\n return row;\n }\n\n #requestToSQL(\n request: FetchRequest,\n filters: NoSubqueryCondition \| undefined,\n order: Ordering \| undefined,\n ): SQLQuery {\n return buildSelectQuery(\n this.#table,\n this.#columns,\n request.constraint,\n filters,\n order,\n request.reverse,\n request.start,\n request.multiConstraints,\n );\n }\n}\n\nfunction getUniqueIndexes(\n db: Database,\n tableName: string,\n): Map<string, Set<string>> {\n const sqlAndBindings = format(\n sql`\n SELECT idx.name, json_group_array(col.name) as columnsJSON\n FROM sqlite_master as idx\n JOIN pragma_index_list(idx.tbl_name) AS info ON info.name = idx.name\n JOIN pragma_index_info(idx.name) as col\n WHERE idx.tbl_name = ${tableName} AND\n idx.type = 'index' AND \n info.\"unique\" != 0\n GROUP BY idx.name\n ORDER BY idx.name`,\n );\n const stmt = db.prepare(sqlAndBindings.text);\n const indexes = stmt.all<{columnsJSON: string}>(...sqlAndBindings.values);\n return new Map(\n indexes.map(({columnsJSON}) => {\n const columns = JSON.parse(columnsJSON);\n const set = new Set<string>(columns);\n return [JSON.stringify(columns.sort()), set];\n }),\n );\n}\n\nexport function toSQLiteTypes(\n columns: readonly string[],\n row: Row,\n columnTypes: Record<string, SchemaValue>,\n): readonly unknown[] {\n return columns.map(col => toSQLiteType(row[col], columnTypes[col].type));\n}\n\nexport function toSQLiteTypeName(type: ValueType) {\n switch (type) {\n case 'boolean':\n return 'INTEGER';\n case 'number':\n return 'REAL';\n case 'string':\n return 'TEXT';\n case 'null':\n return 'NULL';\n case 'json':\n return 'TEXT';\n }\n}\n\nexport function fromSQLiteTypes(\n valueTypes: Record<string, SchemaValue>,\n row: Row,\n tableName: string,\n): Row {\n const newRow: Writable<Row> = {};\n for (const key of Object.keys(row)) {\n const valueType = valueTypes[key];\n if (valueType === undefined) {\n const columnList = Object.keys(valueTypes).sort().join(', ');\n throw new Error(\n `Invalid column \"${key}\" for table \"${tableName}\". Synced columns include ${columnList}`,\n );\n }\n newRow[key] = fromSQLiteType(valueType.type, row[key], key, tableName);\n }\n return newRow;\n}\n\nfunction fromSQLiteType(\n valueType: ValueType,\n v: Value,\n column: string,\n tableName: string,\n): Value {\n if (v === null) {\n return null;\n }\n switch (valueType) {\n case 'boolean':\n return !!v;\n case 'number':\n case 'string':\n case 'null':\n if (typeof v === 'bigint') {\n if (v > Number.MAX_SAFE_INTEGER \|\| v < Number.MIN_SAFE_INTEGER) {\n throw new UnsupportedValueError(\n `value ${v} (in ${tableName}.${column}) is outside of supported bounds`,\n );\n }\n return Number(v);\n }\n return v;\n case 'json':\n try {\n return JSON.parse(v as string);\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n throw new UnsupportedValueError(\n `Failed to parse JSON for ${tableName}.${column}: ${errorMessage}`,\n {cause: error},\n );\n }\n }\n}\n\nexport class UnsupportedValueError extends Error {}\n\nfunction canUseUpdate(\n oldRow: Row,\n row: Row,\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n): boolean {\n for (const pk of primaryKey) {\n if (oldRow[pk] !== row[pk]) {\n return false;\n }\n }\n return Object.keys(columns).length > primaryKey.length;\n}\n\nfunction nonPrimaryValues(\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n row: Row,\n): Iterable<unknown> {\n return nonPrimaryKeys(columns, primaryKey).map(c =>\n toSQLiteType(row[c], columns[c].type),\n );\n}\n\nfunction nonPrimaryKeys(\n columns: Record<string, SchemaValue>,\n primaryKey: PrimaryKey,\n) {\n return Object.keys(columns).filter(c => !primaryKey.includes(c));\n}\n\nfunction generateWithYields(stream: Stream<Node>, shouldYield: () => boolean) {\n for (const n of stream) {\n if (shouldYield()) {\n yield 'yield';\n }\n yield n;\n }\n}\n"],"mappings":";;;;;;;;;;;;AA0DA,IAAI,aAAa;;;;;;;;;;;;;;;AAgBjB,IAAa,cAAb,MAA2C;CACzC,2BAAoB,IAAI,QAA8B;CACtD,eAAsC,CAAC;CACvC;CACA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;;;;;;;CAQb,YACE,YACA,WACA,IACA,WACA,SACA,YACA,oBAAoB,OACpB;EACA,KAAKO,MAAM;EACX,KAAKD,aAAa;EAClB,KAAKJ,SAAS;EACd,KAAKC,WAAW;EAChB,KAAKC,iBAAiB,iBAAiB,IAAI,SAAS;EACpD,KAAKC,cAAc;EACnB,KAAKI,SAAS,KAAKC,kBAAkB,EAAE;EACvC,KAAKF,eAAe;EAEpB,OACE,KAAKJ,eAAe,IAAI,KAAK,UAAU,WAAW,SAAS,CAAC,CAAC,GAC7D,eAAe,WAAW,8BAC5B;CACF;CAEA,IAAI,cAAc;EAChB,OAAO;GACL,MAAM,KAAKF;GACX,SAAS,KAAKC;GACd,YAAY,KAAKE;EACnB;CACF;;;;;CAMA,MAAM,IAAc;EAClB,KAAKI,SAAS,KAAKC,kBAAkB,EAAE;CACzC;CAEA,kBAAkB,IAAc;EAC9B,MAAM,SAAS,KAAKV,SAAS,IAAI,EAAE;EACnC,IAAI,QACF,OAAO;EAGT,MAAM,QAAQ;GACZ,OAAO,IAAI,eAAe,EAAE;GAC5B,QAAQ,GAAG,QACT,QACE,GAAG,eAAe,IAAI,MAAM,KAAKE,MAAM,EAAE,IAAI,IAAI,KAC/C,OAAO,KAAK,KAAKC,QAAQ,EAAE,KAAI,MAAK,IAAI,MAAM,CAAC,CAAC,GAChD,IACF,EAAE,YAAY,IAAI,sBAChB,MAAM,KAAK,EAAC,QAAQ,OAAO,KAAK,KAAKA,QAAQ,EAAE,OAAM,CAAC,EACnD,KAAK,GAAG,EACR,KAAK,GAAG,CACb,EAAE,EACJ,CACF;GACA,QAAQ,GAAG,QACT,QACE,GAAG,eAAe,IAAI,MAAM,KAAKD,MAAM,EAAE,SAAS,IAAI,KACpD,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF;GAEA,QACE,OAAO,KAAK,KAAKF,QAAQ,EAAE,SAAS,KAAKE,YAAY,SACjD,GAAG,QACD,QACE,GAAG,UAAU,IAAI,MAAM,KAAKH,MAAM,EAAE,OAAO,IAAI,KAC7C,eAAe,KAAKC,UAAU,KAAKE,WAAW,EAAE,KAC9C,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAC1B,GACA,GACF,EAAE,SAAS,IAAI,KACb,KAAKA,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF,IACA,KAAA;GACN,aAAa,GAAG,QACd,QACE,GAAG,6BAA6B,IAAI,MAClC,KAAKH,MACP,EAAE,SAAS,IAAI,KACb,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,EAAE,SACJ,CACF;GACA,aAAa,GAAG,QACd,QACE,GAAG,iBAAiB,IAAI,MAAM,KAAKH,MAAM,EAAE,SAAS,IAAI,KACtD,KAAKG,YAAY,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GAChD,OACF,GACF,CACF;EACF;EACA,KAAKL,SAAS,IAAI,IAAI,KAAK;EAC3B,OAAO;CACT;CAEA,IAAIW,cAAc;EAChB,OAAO,IAAI,KACT,OAAO,KAAK,KAAKR,QAAQ,EAAE,KAAI,MAAK,IAAI,MAAM,CAAC,CAAC,GAChD,GAAG,GACL;CACF;CAEA,WAAW,YAAwB,WAAkC;EACnE,OAAO;GACL,WAAW,KAAKD;GAChB,SAAS,KAAKC;GACd,YAAY,KAAKE;GACjB,MAAM,YAAY,KAAA,IAAY,WAAW;GACzC,eAAe,CAAC;GAChB,UAAU;GACV,QAAQ;GACR,aAAa,WAAW;EAC1B;CACF;CAEA,QACE,MACA,SACA,eACA,OACA;EACA,MAAM,qBAAqB,iBAAiB,OAAO;EACnD,MAAM,YAAY,SAAS,KAAA;EAG3B,MAAM,iBAA2B,KAAKA,YAAY,KAAI,MAAK,CAAC,GAAG,KAAK,CAAC;EAErE,MAAM,QAAqB;GACzB,iBAAiB;GACjB,QAAO,QAAO,KAAKO,OAAO,KAAK,UAAU;GACzC,YAAW,WAAU;IACnB,WAAW,SAAS;GACtB;GACA,eAAe;IACb,MAAM,MAAM,KAAKX,aAAa,QAAQ,UAAU;IAChD,OAAO,QAAQ,IAAI,sBAAsB;IACzC,KAAKA,aAAa,OAAO,KAAK,CAAC;GACjC;GACA,qBAAqB,CAAC,mBAAmB;EAC3C;EAEA,MAAM,aAAyB;GAC7B;GACA;GACA,QAAQ,KAAA;GACR;GACA;GACA,SAAS,mBAAmB,UACxB;IACE,WAAW,mBAAmB;IAC9B,WAAW,gBAAgB,mBAAmB,OAAO;GACvD,IACA,KAAA;GACJ,aAAa,OAAO,eAAe,IAAI,IAAI,eAAe,cAAc;GACxE,iBAAiB;EACnB;EACA,MAAM,SAAS,KAAKY,WAAW,YAAY,SAAS;EACpD,IAAI,CAAC,WACH,yBAAyB,MAAM,KAAKR,WAAW;EAGjD,KAAKJ,aAAa,KAAK,UAAU;EACjC,OAAO;CACT;CAEA,YAAY,KAAe;EACzB,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,EAAE,KAAK,CAAC,KAAK,WAAW,CACxC,KACA,aAAa,OAAO,KAAKE,SAAS,KAAK,IAAI,CAC7C,CAAC,CACH;CACF;CAEA,CAACS,OAAO,KAAmB,YAAgD;EACzE,MAAM,EAAC,MAAM,UAAS;EAGtB,MAAM,iBAAiB,OADT,KAAKE,cAAc,KAAK,WAAW,SAAS,WAAW,IACvC,CAAK;EAEnC,MAAM,kBAAkB,KAAKL,OAAO,MAAM,IAAI,eAAe,IAAI;EACjE,gBAAgB,UAAU,aAAa,IAAI;EAC3C,MAAM,cAAc,gBAAgB,UAAU,QAC5C,GAAG,eAAe,MACpB;EACA,IAAI;GACF,OAAO,UAAU,KAAKP,QAAQ,eAAe,IAAI;GAEjD,IAAI,MAAM;IACR,MAAM,aAAa,eAAe,MAAM,IAAI,OAAO;IACnD,OAAO,kBACL,mBACE,oBACE,IAAI,OAAO,KACX,KAAKa,oBACH,KAAKZ,UACL,aACA,eAAe,MACf,KACF,GACA,IAAI,YACJ,KAAKa,UACL,WAAW,iBACX,YAIA,YACA,WAAW,SAAS,WACpB,IAAI,gBACN,GACA,KAAKR,YACP,GACA,IAAI,OACJ,UACF;GACF,OACE,OAAO,mBACL,6BACE,KAAKO,oBACH,KAAKZ,UACL,aACA,eAAe,MACf,KACF,GACA,IAAI,YACJ,KAAKa,UACL,WAAW,iBACX,KAAKX,aACL,WAAW,SAAS,WACpB,IAAI,gBACN,GACA,KAAKG,YACP;EAEJ,UAAU;GAER,YAAY,SAAS;GACrB,IAAI,OAAO;IACT,IAAI,cAAc;IAClB,MAAM,YAAsB,CAAC;IAC7B,KAAK,IAAI,IAAI,IAAK,KAAK;KACrB,MAAM,SAAS,gBAAgB,UAAU,WACvC,GACA,gBAAgB,wBAChB,CACF;KACA,IAAI,WAAW,KAAA,GACb;KAEF,eAAe,OAAO,MAAM;KAC5B,MAAM,UAAU,gBAAgB,UAAU,WACxC,GACA,gBAAgB,yBAChB,CACF;KACA,IAAI,OAAO,YAAY,YAAY,QAAQ,SAAS,GAClD,UAAU,KAAK,OAAO;IAE1B;IACA,IAAI,gBAAgB,GAClB,MAAM,aAAa,KAAKN,QAAQ,eAAe,MAAM,WAAW;IAElE,IAAI,UAAU,SAAS,GACrB,MAAM,cAAc,KAAKA,QAAQ,eAAe,MAAM,SAAS;IAEjE,gBAAgB,UAAU,gBAAgB;GAC5C;GACA,KAAKO,OAAO,MAAM,OAAO,eAAe;EAC1C;CACF;CAEA,CAACM,oBACC,YACA,aACA,OACA,OACuB;EACvB,IAAI;EACJ,GAAG;GACD,SAAS,YACP,KAAKR,KACL,EAAE,YACF,KAAKD,WAAW,mBACV,YAAY,KAAK,GACvB,KAAKA,WAAW,wBAEd,uCAAuC,MAAM,4BACjD;GACA,IAAI,OAAO,MACT;GAEF,MAAM,MAAM,gBAAgB,YAAY,OAAO,OAAO,KAAKJ,MAAM;GACjE,OAAO,UAAU,KAAKA,QAAQ,OAAO,GAAG;GACxC,MAAM;EACR,SAAS,CAAC,OAAO;CACnB;CAEA,CAAC,KAAK,QAAuC;EAC3C,KAAK,MAAM,UAAU,KAAK,QAAQ,MAAM,GACtC,IAAI,WAAW,SACb,MAAM;CAGZ;CAEA,CAAC,QAAQ,QAAsB;EAC7B,MAAM,UAAU,QACd,KAAKO,OAAO,YAAY,IACtB,GAAG,cAAc,KAAKJ,aAAa,KAAK,KAAKF,QAAQ,CACvD,GAAG,WAAW;EAChB,MAAM,cAAc,MAA4B,KAAKa,WAAW;EAChE,MAAM,eAAe,MAAoB,KAAKC,aAAa,CAAC;EAE5D,OAAO,6BACL,KAAKhB,cACL,QACA,QACA,YACA,mBACM,EAAE,KAAKiB,UACf;CACF;CAEA,aAAa,QAAsB;EACjC,QAAQ,OAAO,IAAf;GACE,KAAK;IACH,KAAKT,OAAO,OAAO,IACjB,GAAG,cAED,OAAO,KAAK,KAAKN,QAAQ,GACzB,OAAO,IACP,KAAKA,QACP,CACF;IACA;GACF,KAAK;IACH,KAAKM,OAAO,OAAO,IACjB,GAAG,cACD,KAAKJ,aACL,OAAO,IACP,KAAKF,QACP,CACF;IACA;GACF,KAAK;IAEH,IACE,aACE,OAAO,IACP,OAAO,IACP,KAAKA,UACL,KAAKE,WACP,GACA;KACA,MAAM,YAAY;MAChB,GAAG,OAAO;MACV,GAAG,OAAO;KACZ;KACA,MAAM,SAAS,CACb,GAAG,iBAAiB,KAAKF,UAAU,KAAKE,aAAa,SAAS,GAC9D,GAAG,cAAc,KAAKA,aAAa,WAAW,KAAKF,QAAQ,CAC7D;KACA,KAAK,KAAKM,OAAO,MAAM,EAAE,IAAI,MAAM;IACrC,OAAO;KACL,KAAKA,OAAO,OAAO,IACjB,GAAG,cACD,KAAKJ,aACL,OAAO,IACP,KAAKF,QACP,CACF;KACA,KAAKM,OAAO,OAAO,IACjB,GAAG,cACD,OAAO,KAAK,KAAKN,QAAQ,GACzB,OAAO,IACP,KAAKA,QACP,CACF;IACF;IAEA;GAEF,SACE,YAAY,MAAM;EACtB;CACF;CAEA,mCAAmB,IAAI,IAAoB;CAE3C,YAAY,SAA2B;EACrC,MAAM,YAAY,KAAK,UAAU,OAAO;EACxC,IAAI,OAAO,KAAKgB,iBAAiB,IAAI,SAAS;EAC9C,IAAI,CAAC,MAAM;GACT,OAAO,QACL,GAAG,UAAU,KAAKR,YAAY,QAAQ,IAAI,MACxC,KAAKT,MACP,EAAE,SAAS,IAAI,KACb,QAAQ,KAAI,MAAK,GAAG,GAAG,IAAI,MAAM,CAAC,EAAE,GAAG,GACvC,GAAG,MACL,GACF;GACA,KAAKiB,iBAAiB,IAAI,WAAW,IAAI;EAC3C;EACA,OAAO;CACT;;;;;;;;CASA,OAAO,QAA8B;EACnC,MAAM,UAAU,OAAO,KAAK,MAAM;EAElC,MAAM,OAAO,KAAKC,YAAY,OAAO;EACrC,MAAM,MAAM,KAAKX,OAAO,MAAM,IAAI,OAAM,WACtC,OAAO,UACJ,aAAa,IAAI,EACjB,IAAS,GAAG,cAAc,SAAS,QAAQ,KAAKN,QAAQ,CAAC,CAC9D;EACA,IAAI,KACF,OAAO,gBAAgB,KAAKA,UAAU,KAAK,KAAKD,MAAM;EAExD,OAAO;CACT;CAEA,cACE,SACA,SACA,OACU;EACV,OAAO,iBACL,KAAKA,QACL,KAAKC,UACL,QAAQ,YACR,SACA,OACA,QAAQ,SACR,QAAQ,OACR,QAAQ,gBACV;CACF;AACF;AAEA,SAAS,iBACP,IACA,WAC0B;CAC1B,MAAM,iBAAiB,OACrB,GAAG;;;;;6BAKsB,UAAU;;;;wBAKrC;CAEA,MAAM,UADO,GAAG,QAAQ,eAAe,IACvB,EAAK,IAA2B,GAAG,eAAe,MAAM;CACxE,OAAO,IAAI,IACT,QAAQ,KAAK,EAAC,kBAAiB;EAC7B,MAAM,UAAU,KAAK,MAAM,WAAW;EACtC,MAAM,MAAM,IAAI,IAAY,OAAO;EACnC,OAAO,CAAC,KAAK,UAAU,QAAQ,KAAK,CAAC,GAAG,GAAG;CAC7C,CAAC,CACH;AACF;AAEA,SAAgB,cACd,SACA,KACA,aACoB;CACpB,OAAO,QAAQ,KAAI,QAAO,aAAa,IAAI,MAAM,YAAY,KAAK,IAAI,CAAC;AACzE;AAiBA,SAAgB,gBACd,YACA,KACA,WACK;CACL,MAAM,SAAwB,CAAC;CAC/B,KAAK,MAAM,OAAO,OAAO,KAAK,GAAG,GAAG;EAClC,MAAM,YAAY,WAAW;EAC7B,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,aAAa,OAAO,KAAK,UAAU,EAAE,KAAK,EAAE,KAAK,IAAI;GAC3D,MAAM,IAAI,MACR,mBAAmB,IAAI,eAAe,UAAU,4BAA4B,YAC9E;EACF;EACA,OAAO,OAAO,eAAe,UAAU,MAAM,IAAI,MAAM,KAAK,SAAS;CACvE;CACA,OAAO;AACT;AAEA,SAAS,eACP,WACA,GACA,QACA,WACO;CACP,IAAI,MAAM,MACR,OAAO;CAET,QAAQ,WAAR;EACE,KAAK,WACH,OAAO,CAAC,CAAC;EACX,KAAK;EACL,KAAK;EACL,KAAK;GACH,IAAI,OAAO,MAAM,UAAU;IACzB,IAAI,IAAI,OAAO,oBAAoB,IAAI,OAAO,kBAC5C,MAAM,IAAI,sBACR,SAAS,EAAE,OAAO,UAAU,GAAG,OAAO,iCACxC;IAEF,OAAO,OAAO,CAAC;GACjB;GACA,OAAO;EACT,KAAK,QACH,IAAI;GACF,OAAO,KAAK,MAAM,CAAW;EAC/B,SAAS,OAAO;GAGd,MAAM,IAAI,sBACR,4BAA4B,UAAU,GAAG,OAAO,IAFhD,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAGrD,EAAC,OAAO,MAAK,CACf;EACF;CACJ;AACF;AAEA,IAAa,wBAAb,cAA2C,MAAM,CAAC;AAElD,SAAS,aACP,QACA,KACA,SACA,YACS;CACT,KAAK,MAAM,MAAM,YACf,IAAI,OAAO,QAAQ,IAAI,KACrB,OAAO;CAGX,OAAO,OAAO,KAAK,OAAO,EAAE,SAAS,WAAW;AAClD;AAEA,SAAS,iBACP,SACA,YACA,KACmB;CACnB,OAAO,eAAe,SAAS,UAAU,EAAE,KAAI,MAC7C,aAAa,IAAI,IAAI,QAAQ,GAAG,IAAI,CACtC;AACF;AAEA,SAAS,eACP,SACA,YACA;CACA,OAAO,OAAO,KAAK,OAAO,EAAE,QAAO,MAAK,CAAC,WAAW,SAAS,CAAC,CAAC;AACjE;AAEA,UAAU,mBAAmB,QAAsB,aAA4B;CAC7E,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,YAAY,GACd,MAAM;EAER,MAAM;CACR;AACF"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "1.10.0-canary.1",
+  "version": "1.10.0-canary.3",
   "description": "Zero is a web framework for serverless web development.",
   "homepage": "https://zero.rocicorp.dev",
   "bugs": {

package/out/zero-cache/src/server/backup-watermark-poller.js DELETED Viewed

@@ -1,31 +0,0 @@
-import { must } from "../../../shared/src/must.js";
-import { parentWorker, singleProcessMode } from "../types/processes.js";
-import { getZeroConfig } from "../config/zero-config.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
-import { createLogContext } from "./logging.js";
-import { VfsBackupWatermarkPoller } from "../services/litestream/vfs-watermark-poller.js";
-import { LogContext, consoleLogSink } from "@rocicorp/logger";
-//#region ../zero-cache/src/server/backup-watermark-poller.ts
-var lc = new LogContext("info", {}, consoleLogSink);
-async function runWorker(parent, env, ...argv) {
-	const config = getZeroConfig({
-		env,
-		argv
-	});
-	lc = createLogContext(config, "backup-watermark-poller");
-	const { litestream, replica } = config;
-	parent.send(["ready", { ready: true }]);
-	await runUntilKilled(lc, parent, new VfsBackupWatermarkPoller(lc, parent, {
-		localDbFile: replica.file,
-		backupURL: must(litestream.backupURL, "Missing --litestream-backup-url for backup watermark reader"),
-		endpoint: litestream.endpoint,
-		region: litestream.region,
-		extensionPath: litestream.vfsExtensionPath,
-		remotePollIntervalMs: litestream.vfsProbeIntervalMs
-	}));
-}
-if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
-//#endregion
-export { runWorker as default };
-//# sourceMappingURL=backup-watermark-poller.js.map

package/out/zero-cache/src/server/backup-watermark-poller.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"backup-watermark-poller.js","names":[],"sources":["../../../../../zero-cache/src/server/backup-watermark-poller.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getZeroConfig} from '../config/zero-config.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {VfsBackupWatermarkPoller} from '../services/litestream/vfs-watermark-poller.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\n\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getZeroConfig({env, argv});\n lc = createLogContext(config, 'backup-watermark-poller');\n const {litestream, replica} = config;\n\n parent.send(['ready', {ready: true}]);\n\n await runUntilKilled(\n lc,\n parent,\n new VfsBackupWatermarkPoller(lc, parent, {\n localDbFile: replica.file,\n backupURL: must(\n litestream.backupURL,\n 'Missing --litestream-backup-url for backup watermark reader',\n ),\n endpoint: litestream.endpoint,\n region: litestream.region,\n extensionPath: litestream.vfsExtensionPath,\n remotePollIntervalMs: litestream.vfsProbeIntervalMs,\n }),\n );\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;AAYA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,cAAc;EAAC;EAAK;CAAI,CAAC;CACxC,KAAK,iBAAiB,QAAQ,yBAAyB;CACvD,MAAM,EAAC,YAAY,YAAW;CAE9B,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,MAAM,eACJ,IACA,QACA,IAAI,yBAAyB,IAAI,QAAQ;EACvC,aAAa,QAAQ;EACrB,WAAW,KACT,WAAW,WACX,6DACF;EACA,UAAU,WAAW;EACrB,QAAQ,WAAW;EACnB,eAAe,WAAW;EAC1B,sBAAsB,WAAW;CACnC,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts DELETED Viewed

@@ -1,57 +0,0 @@
-import type { LogContext, LogLevel } from '@rocicorp/logger';
-import { type Worker } from '../../types/processes.ts';
-import type { Service } from '../service.ts';
-type LoadableDatabase = {
-    loadExtension(path: string): unknown;
-    close(): void;
-};
-type QueryDatabase = {
-    prepare(sql: string): {
-        get<T>(): T;
-    };
-    close(): void;
-};
-export type BackupWatermarkUpdate = [
-    'backupWatermarkUpdate',
-    {
-        watermark: string;
-        writeTimeMs: number;
-        backupTimeMs: number;
-    }
-];
-export type VfsBackupWatermarkPollerEnvOptions = {
-    readonly backupURL: string;
-    readonly remotePollIntervalMs: number;
-    readonly endpoint?: string | undefined;
-    readonly region?: string | undefined;
-    readonly logLevel?: LogLevel | undefined;
-    readonly logFile?: string | undefined;
-};
-export type VfsBackupWatermarkPollerOptions = {
-    readonly localDbFile: string;
-    readonly backupURL: string;
-    readonly endpoint?: string | undefined;
-    readonly region?: string | undefined;
-    readonly extensionPath: string;
-    readonly remotePollIntervalMs: number;
-};
-type VfsBackupWatermarkPollerDeps = {
-    readonly loaderFactory?: (() => LoadableDatabase) | undefined;
-    readonly databaseFactory?: ((lc: LogContext, uri: string) => QueryDatabase) | undefined;
-};
-export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
-    readonly backupURL: string;
-    readonly endpoint?: string | undefined;
-    readonly region?: string | undefined;
-}): string;
-export declare function getVfsEnv(options: VfsBackupWatermarkPollerEnvOptions): NodeJS.ProcessEnv;
-export declare class VfsBackupWatermarkPoller implements Service {
-    #private;
-    readonly id = "backup-watermark-poller";
-    constructor(lc: LogContext, parent: Worker, options: VfsBackupWatermarkPollerOptions, deps?: VfsBackupWatermarkPollerDeps);
-    checkWatermarks(): void;
-    run(): Promise<void>;
-    stop(): Promise<void>;
-}
-export {};
-//# sourceMappingURL=vfs-watermark-poller.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"vfs-watermark-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-poller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAErD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAI3C,KAAK,gBAAgB,GAAG;IACtB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG;QAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;KAAC,CAAC;IACpC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAeF,MAAM,MAAM,qBAAqB,GAAG;IAClC,uBAAuB;IACvB;QACE,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB;CACF,CAAC;AAUF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAErC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,gBAAgB,CAAC,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,eAAe,CAAC,EACrB,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,KAAK,aAAa,CAAC,GAChD,SAAS,CAAC;CACf,CAAC;AAEF,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT;AAED,wBAAgB,SAAS,CACvB,OAAO,EAAE,kCAAkC,GAC1C,MAAM,CAAC,UAAU,CAWnB;AAKD,qBAAa,wBAAyB,YAAW,OAAO;;IACtD,QAAQ,CAAC,EAAE,6BAA6B;gBAatC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,+BAA+B,EACxC,IAAI,GAAE,4BAAiC;IAgEzC,eAAe;IAiDT,GAAG;IAkBT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAItB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js DELETED Viewed

@@ -1,137 +0,0 @@
-import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
-import { parse, valita_exports } from "../../../../shared/src/valita.js";
-import "../../types/processes.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import { RunningState } from "../running-state.js";
-import Sqlite3Database from "@rocicorp/zero-sqlite3";
-//#region ../zero-cache/src/services/litestream/vfs-watermark-poller.ts
-var DEFAULT_VFS_LOG_LEVEL = "info";
-var watermarkRowSchema = valita_exports.object({
-	watermark: valita_exports.string(),
-	writeTimeMs: valita_exports.number()
-});
-var watermarkQuerySchema = watermarkRowSchema.extend({ backupTime: valita_exports.string() });
-function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
-	if (endpoint === void 0 && region === void 0) return backupURL;
-	const url = new URL(backupURL);
-	if (url.protocol !== "s3:") return backupURL;
-	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
-	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
-	return url.toString();
-}
-function getVfsEnv(options) {
-	return {
-		LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),
-		LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,
-		LITESTREAM_LOG_FILE: options.logFile,
-		LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`
-	};
-}
-var MIN_CHECK_INTERVAL_MS = 1e3;
-var MAX_CHECK_INTERVAL_MS = 6e4;
-var VfsBackupWatermarkPoller = class {
-	id = "backup-watermark-poller";
-	#runningState = new RunningState(this.id);
-	#lc;
-	#parent;
-	#loader;
-	#localDb;
-	#openRemote;
-	#remoteDb = null;
-	#lastRemoteWatermark = null;
-	constructor(lc, parent, options, deps = {}) {
-		lc = lc.withContext("component", "vfs-watermark-poller");
-		this.#lc = lc;
-		this.#parent = parent;
-		const { databaseFactory } = deps;
-		const openDb = (url, type, schema) => {
-			try {
-				const db = databaseFactory ? databaseFactory(lc, url) : new Database(lc, url, { readonly: true });
-				const stmt = db.prepare(type === "local" ? `
-          SELECT stateVersion AS watermark, writeTimeMs FROM "_zero.replicationState"` : `
-          SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM "_zero.replicationState"`);
-				return {
-					getWatermark: () => parse(stmt.get(), schema),
-					close: () => db.close()
-				};
-			} catch (e) {
-				lc.error?.(`error opening ${url}`, e);
-				throw e;
-			}
-		};
-		this.#localDb = openDb(options.localDbFile, "local", watermarkRowSchema);
-		const loaderFactory = deps.loaderFactory ?? (() => new Sqlite3Database(":memory:"));
-		const remoteDbURL = `file:zero-backup.db?${new URLSearchParams({
-			vfs: "litestream",
-			mode: "ro",
-			replica_url: buildLitestreamVfsReplicaURL(options),
-			poll_interval: `${options.remotePollIntervalMs}ms`
-		}).toString()}`;
-		lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);
-		this.#loader = loaderFactory();
-		this.#loader.loadExtension(options.extensionPath);
-		this.#openRemote = () => openDb(remoteDbURL, "remote", watermarkQuerySchema);
-	}
-	checkWatermarks() {
-		const local = this.#localDb.getWatermark();
-		if (local.watermark !== this.#lastRemoteWatermark?.watermark) {
-			if (!this.#remoteDb) {
-				if (this.#lastRemoteWatermark) this.#lc.debug?.(`local and remote watermarks differ. resuming backup polling`, {
-					local,
-					remote: this.#lastRemoteWatermark
-				});
-				this.#remoteDb = this.#openRemote();
-			}
-			const remote = this.#remoteDb.getWatermark();
-			if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {
-				this.#lc.debug?.(`backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`, {
-					local,
-					remote
-				});
-				this.#lastRemoteWatermark = remote;
-				const { watermark, writeTimeMs, backupTime } = remote;
-				const backupTimeMs = new Date(backupTime).getTime();
-				this.#parent.send(["backupWatermarkUpdate", {
-					watermark,
-					writeTimeMs,
-					backupTimeMs
-				}]);
-			}
-		}
-		if (local.watermark === this.#lastRemoteWatermark?.watermark) {
-			if (this.#remoteDb) {
-				this.#lc.debug?.(`local and remote watermarks match. pausing backup polling`, {
-					local,
-					remote: this.#lastRemoteWatermark
-				});
-				this.#remoteDb.close();
-				this.#remoteDb = null;
-			}
-		}
-	}
-	async run() {
-		this.#lc.info?.(`starting backup polling`);
-		let interval = MIN_CHECK_INTERVAL_MS;
-		while (this.#runningState.shouldRun()) {
-			try {
-				this.checkWatermarks();
-				interval = MIN_CHECK_INTERVAL_MS;
-			} catch (e) {
-				this.#lc.warn?.(`error checking local and remote watermarks`, e);
-				interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);
-			}
-			await this.#runningState.sleep(interval);
-		}
-		this.#remoteDb?.close();
-		this.#localDb.close();
-		this.#loader.close();
-	}
-	stop() {
-		this.#runningState.stop(this.#lc);
-		return promiseVoid;
-	}
-};
-//#endregion
-export { VfsBackupWatermarkPoller, getVfsEnv };
-//# sourceMappingURL=vfs-watermark-poller.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map DELETED Viewed

@@ -1 +0,0 @@

- {"version":3,"file":"vfs-watermark-poller.js","names":["#runningState","#lc","#parent","#loader","#localDb","#openRemote","#lastRemoteWatermark","#remoteDb"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-poller.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n loadExtension(path: string): unknown;\n close(): void;\n};\n\ntype QueryDatabase = {\n prepare(sql: string): {get<T>(): T};\n close(): void;\n};\n\nconst watermarkRowSchema = v.object({\n watermark: v.string(),\n writeTimeMs: v.number(),\n});\n\nconst watermarkQuerySchema = watermarkRowSchema.extend({\n backupTime: v.string(),\n});\n\ntype WatermarkQueryResult = v.Infer<typeof watermarkQuerySchema>;\n\ntype WatermarkRow = v.Infer<typeof watermarkRowSchema>;\n\nexport type BackupWatermarkUpdate = [\n 'backupWatermarkUpdate',\n {\n watermark: string;\n writeTimeMs: number;\n backupTimeMs: number;\n },\n];\n\ntype WatermarkReader<Result extends WatermarkRow> = {\n getWatermark(): Result;\n close(): void;\n};\n\n// Values that must be passed via the ENV to the original process (worker).\n// Some of these will be configurable via the db URI once\n// https://github.com/benbjohnson/litestream/issues/1150 is released.\nexport type VfsBackupWatermarkPollerEnvOptions = {\n readonly backupURL: string;\n readonly remotePollIntervalMs: number;\n readonly endpoint?: string | undefined;\n readonly region?: string | undefined;\n\n readonly logLevel?: LogLevel | undefined;\n readonly logFile?: string | undefined;\n};\n\nexport type VfsBackupWatermarkPollerOptions = {\n readonly localDbFile: string;\n readonly backupURL: string;\n readonly endpoint?: string | undefined;\n readonly region?: string | undefined;\n\n readonly extensionPath: string;\n readonly remotePollIntervalMs: number;\n};\n\ntype VfsBackupWatermarkPollerDeps = {\n readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n readonly databaseFactory?:\n | ((lc: LogContext, uri: string) => QueryDatabase)\n | undefined;\n};\n\nexport function buildLitestreamVfsReplicaURL({\n backupURL,\n endpoint,\n region,\n}: {\n readonly backupURL: string;\n readonly endpoint?: string | undefined;\n readonly region?: string | undefined;\n}): string {\n if (endpoint === undefined && region === undefined) {\n return backupURL;\n }\n\n const url = new URL(backupURL);\n if (url.protocol !== 's3:') {\n return backupURL;\n }\n\n if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n url.searchParams.set('endpoint', endpoint);\n }\n if (region !== undefined && !url.searchParams.has('region')) {\n url.searchParams.set('region', region);\n }\n return url.toString();\n}\n\nexport function getVfsEnv(\n options: VfsBackupWatermarkPollerEnvOptions,\n): NodeJS.ProcessEnv {\n return {\n LITESTREAM_REPLICA_URL: buildLitestreamVfsReplicaURL(options),\n LITESTREAM_LOG_LEVEL: options.logLevel ?? DEFAULT_VFS_LOG_LEVEL,\n LITESTREAM_LOG_FILE: options.logFile,\n\n // Supported by the rocicorp fork: https://github.com/rocicorp/litestream/pull/20\n // TODO: Remove and use official build when\n // https://github.com/benbjohnson/litestream/pull/1157 is released.\n LITESTREAM_POLL_INTERVAL: `${options.remotePollIntervalMs}ms`,\n };\n}\n\nconst MIN_CHECK_INTERVAL_MS = 1_000;\nconst MAX_CHECK_INTERVAL_MS = 60_000;\n\nexport class VfsBackupWatermarkPoller implements Service {\n readonly id = 'backup-watermark-poller';\n readonly #runningState = new RunningState(this.id);\n readonly #lc: LogContext;\n readonly #parent: Worker;\n readonly #loader: LoadableDatabase;\n\n readonly #localDb: WatermarkReader<WatermarkRow>;\n readonly #openRemote: () => WatermarkReader<WatermarkQueryResult>;\n\n #remoteDb: WatermarkReader<WatermarkQueryResult> | null = null;\n #lastRemoteWatermark: WatermarkQueryResult | null = null;\n\n constructor(\n lc: LogContext,\n parent: Worker,\n options: VfsBackupWatermarkPollerOptions,\n deps: VfsBackupWatermarkPollerDeps = {},\n ) {\n lc = lc.withContext('component', 'vfs-watermark-poller');\n this.#lc = lc;\n this.#parent = parent;\n\n const {databaseFactory} = deps;\n const openDb = <Result extends WatermarkRow>(\n url: string,\n type: 'local' | 'remote',\n schema: v.Type<Result>,\n ) => {\n try {\n const db = databaseFactory\n ? databaseFactory(lc, url)\n : new Database(lc, url, {readonly: true});\n const stmt = db.prepare(\n type === 'local'\n ? /*sql*/ `\n SELECT stateVersion AS watermark, writeTimeMs FROM \"_zero.replicationState\"`\n : /*sql*/ `\n SELECT stateVersion AS watermark, writeTimeMs, litestream_time() as backupTime FROM \"_zero.replicationState\"`,\n );\n return {\n getWatermark: () => v.parse(stmt.get(), schema),\n close: () => db.close(),\n } satisfies WatermarkReader<Result>;\n } catch (e) {\n lc.error?.(`error opening ${url}`, e);\n throw e;\n }\n };\n\n // Open the local (non-VFS) connection before loading the litestream VFS\n // extension. Loading the extension registers a process-global\n // `sqlite3_auto_extension` hook that SQLite invokes on every connection\n // opened *afterward* in this process (even ones that don't reference\n // `vfs=litestream`), and a signature bug in that hook currently causes\n // it to fail such connections. Opening #localDb first sidesteps it,\n // since already-open connections aren't affected by the hook.\n this.#localDb = openDb(options.localDbFile, 'local', watermarkRowSchema);\n\n const loaderFactory =\n deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n\n const dbParams = new URLSearchParams({\n vfs: 'litestream',\n mode: 'ro',\n // Support forthcoming in https://github.com/benbjohnson/litestream/issues/1150\n replica_url: buildLitestreamVfsReplicaURL(options),\n poll_interval: `${options.remotePollIntervalMs}ms`,\n });\n // Note: the filename is arbitrary and only referenced in debug logs.\n const remoteDbURL = `file:zero-backup.db?${dbParams.toString()}`;\n\n lc.info?.(`Loading VFS ${remoteDbURL} at ${options.extensionPath}`);\n this.#loader = loaderFactory();\n this.#loader.loadExtension(options.extensionPath);\n\n this.#openRemote = () =>\n openDb(remoteDbURL, 'remote', watermarkQuerySchema);\n }\n\n // public for testing\n checkWatermarks() {\n const local = this.#localDb.getWatermark();\n if (local.watermark !== this.#lastRemoteWatermark?.watermark) {\n if (!this.#remoteDb) {\n if (this.#lastRemoteWatermark) {\n this.#lc.debug?.(\n `local and remote watermarks differ. resuming backup polling`,\n {local, remote: this.#lastRemoteWatermark},\n );\n }\n this.#remoteDb = this.#openRemote();\n }\n const remote = this.#remoteDb.getWatermark();\n if (remote.watermark !== this.#lastRemoteWatermark?.watermark) {\n this.#lc.debug?.(\n `backup watermark advanced from ${this.#lastRemoteWatermark?.watermark} to ${remote.watermark}`,\n {local, remote},\n );\n this.#lastRemoteWatermark = remote;\n const {watermark, writeTimeMs, backupTime} = remote;\n // Note: litestream_time() returns \"latest\" if no LTX files were\n // loaded:\n //\n // https://github.com/benbjohnson/litestream/blob/6d61ef5d007756d62e473daee4c760ac395a55c6/vfs.go#L2448\n //\n // but in that case the watermark query would have failed, so it\n // follows that litestream_time() should always return a valid\n // timestamp if the watermark query itself succeeded.\n const backupTimeMs = new Date(backupTime).getTime();\n this.#parent.send<BackupWatermarkUpdate>([\n 'backupWatermarkUpdate',\n {watermark, writeTimeMs, backupTimeMs},\n ]);\n }\n }\n // Note: This is not an `else`, as we specifically want to re-check\n // this.#lastRemoteWatermark\n if (local.watermark === this.#lastRemoteWatermark?.watermark) {\n if (this.#remoteDb) {\n this.#lc.debug?.(\n `local and remote watermarks match. pausing backup polling`,\n {local, remote: this.#lastRemoteWatermark},\n );\n this.#remoteDb.close();\n this.#remoteDb = null;\n }\n }\n }\n\n async run() {\n this.#lc.info?.(`starting backup polling`);\n let interval = MIN_CHECK_INTERVAL_MS;\n while (this.#runningState.shouldRun()) {\n try {\n this.checkWatermarks();\n interval = MIN_CHECK_INTERVAL_MS;\n } catch (e) {\n this.#lc.warn?.(`error checking local and remote watermarks`, e);\n interval = Math.min(MAX_CHECK_INTERVAL_MS, interval * 2);\n }\n await this.#runningState.sleep(interval);\n }\n this.#remoteDb?.close();\n this.#localDb.close();\n this.#loader.close();\n }\n\n stop(): Promise<void> {\n this.#runningState.stop(this.#lc);\n return promiseVoid;\n }\n}\n"],"mappings":";;;;;;;AASA,IAAM,wBAAwB;AAY9B,IAAM,qBAAqB,eAAE,OAAO;CAClC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO;AACxB,CAAC;AAED,IAAM,uBAAuB,mBAAmB,OAAO,EACrD,YAAY,eAAE,OAAO,EACvB,CAAC;AAkDD,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,SAAgB,UACd,SACmB;CACnB,OAAO;EACL,wBAAwB,6BAA6B,OAAO;EAC5D,sBAAsB,QAAQ,YAAY;EAC1C,qBAAqB,QAAQ;EAK7B,0BAA0B,GAAG,QAAQ,qBAAqB;CAC5D;AACF;AAEA,IAAM,wBAAwB;AAC9B,IAAM,wBAAwB;AAE9B,IAAa,2BAAb,MAAyD;CACvD,KAAc;CACd,gBAAyB,IAAI,aAAa,KAAK,EAAE;CACjD;CACA;CACA;CAEA;CACA;CAEA,YAA0D;CAC1D,uBAAoD;CAEpD,YACE,IACA,QACA,SACA,OAAqC,CAAC,GACtC;EACA,KAAK,GAAG,YAAY,aAAa,sBAAsB;EACvD,KAAKC,MAAM;EACX,KAAKC,UAAU;EAEf,MAAM,EAAC,oBAAmB;EAC1B,MAAM,UACJ,KACA,MACA,WACG;GACH,IAAI;IACF,MAAM,KAAK,kBACP,gBAAgB,IAAI,GAAG,IACvB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;IAC1C,MAAM,OAAO,GAAG,QACd,SAAS,UACG;yFAEA;uHAEd;IACA,OAAO;KACL,oBAAoB,MAAQ,KAAK,IAAI,GAAG,MAAM;KAC9C,aAAa,GAAG,MAAM;IACxB;GACF,SAAS,GAAG;IACV,GAAG,QAAQ,iBAAiB,OAAO,CAAC;IACpC,MAAM;GACR;EACF;EASA,KAAKE,WAAW,OAAO,QAAQ,aAAa,SAAS,kBAAkB;EAEvE,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAU7D,MAAM,cAAc,uBAAuB,IARtB,gBAAgB;GACnC,KAAK;GACL,MAAM;GAEN,aAAa,6BAA6B,OAAO;GACjD,eAAe,GAAG,QAAQ,qBAAqB;EACjD,CAE2C,EAAS,SAAS;EAE7D,GAAG,OAAO,eAAe,YAAY,MAAM,QAAQ,eAAe;EAClE,KAAKD,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKE,oBACH,OAAO,aAAa,UAAU,oBAAoB;CACtD;CAGA,kBAAkB;EAChB,MAAM,QAAQ,KAAKD,SAAS,aAAa;EACzC,IAAI,MAAM,cAAc,KAAKE,sBAAsB,WAAW;GAC5D,IAAI,CAAC,KAAKC,WAAW;IACnB,IAAI,KAAKD,sBACP,KAAKL,IAAI,QACP,+DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IAEF,KAAKC,YAAY,KAAKF,YAAY;GACpC;GACA,MAAM,SAAS,KAAKE,UAAU,aAAa;GAC3C,IAAI,OAAO,cAAc,KAAKD,sBAAsB,WAAW;IAC7D,KAAKL,IAAI,QACP,kCAAkC,KAAKK,sBAAsB,UAAU,MAAM,OAAO,aACpF;KAAC;KAAO;IAAM,CAChB;IACA,KAAKA,uBAAuB;IAC5B,MAAM,EAAC,WAAW,aAAa,eAAc;IAS7C,MAAM,eAAe,IAAI,KAAK,UAAU,EAAE,QAAQ;IAClD,KAAKJ,QAAQ,KAA4B,CACvC,yBACA;KAAC;KAAW;KAAa;IAAY,CACvC,CAAC;GACH;EACF;EAGA,IAAI,MAAM,cAAc,KAAKI,sBAAsB;OAC7C,KAAKC,WAAW;IAClB,KAAKN,IAAI,QACP,6DACA;KAAC;KAAO,QAAQ,KAAKK;IAAoB,CAC3C;IACA,KAAKC,UAAU,MAAM;IACrB,KAAKA,YAAY;GACnB;;CAEJ;CAEA,MAAM,MAAM;EACV,KAAKN,IAAI,OAAO,yBAAyB;EACzC,IAAI,WAAW;EACf,OAAO,KAAKD,cAAc,UAAU,GAAG;GACrC,IAAI;IACF,KAAK,gBAAgB;IACrB,WAAW;GACb,SAAS,GAAG;IACV,KAAKC,IAAI,OAAO,8CAA8C,CAAC;IAC/D,WAAW,KAAK,IAAI,uBAAuB,WAAW,CAAC;GACzD;GACA,MAAM,KAAKD,cAAc,MAAM,QAAQ;EACzC;EACA,KAAKO,WAAW,MAAM;EACtB,KAAKH,SAAS,MAAM;EACpB,KAAKD,QAAQ,MAAM;CACrB;CAEA,OAAsB;EACpB,KAAKH,cAAc,KAAK,KAAKC,GAAG;EAChC,OAAO;CACT;AACF"}