@powersync/service-sync-rules 0.29.10 → 0.31.0

package/dist/sync_plan/plan.d.ts

@@ -0,0 +1,196 @@
+import { BucketPriority } from '../BucketDescription.js';
+import { ParameterLookupScope } from '../HydrationState.js';
+import { ImplicitSchemaTablePattern } from '../TablePattern.js';
+import { SqlExpression } from './expression.js';
+/**
+ * A compiled "sync plan", a description for
+ *
+ *   1. how rows get filtered, mapped and put into buckets.
+ *   2. how rows get filtered, mapped and put into parameter lookups.
+ *   3. how connections use parameter lookups to infer buckets.
+ *
+ * Sync plans can be compiled from sync stream definitions. There is always a single plan for all stream definitions,
+ * which allows re-using parameter lookups if multiple streams have similar subpatterns.
+ *
+ * Additionally, sync plans can be serialized to (and restored from) a simple JSON representation. We store sync plans
+ * in bucket storage and restore them on service startup, allowing us to make changes to the compiler without that
+ * affecting existing deployed sync instances.
+ *
+ * However, this also means that sync plan definitions themselves must have a stable representation. We can add to them,
+ * but we must never alter the semantics for existing serialized plans.
+ */
+export interface SyncPlan {
+    dataSources: StreamDataSource[];
+    buckets: StreamBucketDataSource[];
+    parameterIndexes: StreamParameterIndexLookupCreator[];
+    streams: CompiledSyncStream[];
+}
+/**
+ * Something that processes a source row if it matches filters.
+ */
+export interface TableProcessor {
+    /**
+     * A hash code describing the structure of this processor.
+     *
+     * While these processors are plain JavaScript objects that can be compared by deep equality, this hash code may
+     * improve performance when many such sources need to be compared. Two equal data sources will always have the same
+     * hash code.
+     */
+    hashCode: number;
+    /**
+     * The source table to process into buckets or parameter lookups.
+     */
+    sourceTable: ImplicitSchemaTablePattern;
+    /**
+     * All of these expressions exclusively depend on the {@link sourceTable}.
+     *
+     * All of the filters must evaluate to a "true-ish" value for the row to be processed.
+     */
+    filters: SqlExpression<ColumnSqlParameterValue>[];
+    /**
+     * How to partition rows.
+     *
+     * For data sources, this describes parameter of the bucket being created. For parameter lookups, these form
+     * input values for the lookup.
+     */
+    parameters: PartitionKey[];
+}
+export interface PartitionKey {
+    expr: SqlExpression<ColumnSqlParameterValue>;
+}
+/**
+ * A description for a data source processing rows to replicate.
+ *
+ * A single data source might be used in multiple buckets ({@link StreamBucketDataSource}). In those cases, we'd store
+ * duplicate bucket data but can still share the actual processing logic between multiple streams (to avoid evaluating
+ * the same filters and expressions multiple times).
+ */
+export interface StreamDataSource extends TableProcessor {
+    /**
+     * The name of the output table for evaluated rows.
+     *
+     * If null, the name of the table being evaluated should be used instead.
+     */
+    outputTableName?: string;
+    /**
+     * Output columns describing the row to store in buckets.
+     */
+    columns: ColumnSource[];
+}
+export type ColumnSource = 'star' | {
+    expr: SqlExpression<ColumnSqlParameterValue>;
+    alias: string;
+};
+/**
+ * A mapping describing how {@link StreamDataSource}s are combined into buckets.
+ *
+ * One instance of this always describes a single bucket data source. A stream may consist of multiple such sources if
+ * different variants are used. It is also possible for a bucket data source to be reused between streams in some
+ * instances.
+ */
+export interface StreamBucketDataSource {
+    hashCode: number;
+    /**
+     * A unique name of this source in all streams compiled from a sync rules file.
+     */
+    uniqueName: string;
+    /**
+     * All data sources to put into this bucket.
+     *
+     * Note that all data sources must have the same amount of paremeters, since they would be instantiated to the same
+     * values by a {@link StreamQuerier}.
+     */
+    sources: StreamDataSource[];
+}
+/**
+ * A description for a data source creating parameter lookup values that can be queried when fetching buckets for a
+ * connection.
+ */
+export interface StreamParameterIndexLookupCreator extends TableProcessor {
+    hashCode: number;
+    defaultLookupScope: ParameterLookupScope;
+    /**
+     * Outputs to persist in the lookup.
+     *
+     * Note: In {@link UnscopedEvaluatedParameters}, outputs are called `bucketParameters`. This doesn't apply to sync
+     * streams because the output of parameters might be passed through additional stages or transformed by the querier
+     * before becoming a parameter value.
+     */
+    outputs: SqlExpression<ColumnSqlParameterValue>[];
+}
+export interface StreamOptions {
+    name: string;
+    isSubscribedByDefault: boolean;
+    priority: BucketPriority;
+}
+export interface CompiledSyncStream {
+    stream: StreamOptions;
+    queriers: StreamQuerier[];
+}
+export interface StreamQuerier {
+    /**
+     * Static filters on the subscription, connection or token.
+     *
+     * All of them must match for this querier to be considered for a subscription.
+     */
+    requestFilters: SqlExpression<RequestSqlParameterValue>[];
+    /**
+     * Ordered lookups that need to be evaluated and expanded before we can evaluate {@link sourceInstantiation}.
+     *
+     * Each element of the outer array represents a stage of lookups that can run concurrently.
+     */
+    lookupStages: ExpandingLookup[][];
+    /**
+     * The bucket being resolved by this querier.
+     *
+     * While a querier can have more than one source, it never uses more than one instantiation path. This means that all
+     * sources in {@link StreamBucketDataSource.sources} must have compatible parameters.
+     */
+    bucket: StreamBucketDataSource;
+    sourceInstantiation: ParameterValue[];
+}
+export type SqlParameterValue = ColumnSqlParameterValue | RequestSqlParameterValue;
+/**
+ * A value that resolves to a given column in a row being processed.
+ */
+export interface ColumnSqlParameterValue {
+    column: string;
+}
+export type ConnectionParameterSource = 'auth' | 'subscription' | 'connection';
+/**
+ * A value that resolves to either the current subscription parameters, the JWT of the connecting user or global
+ * request parameters.
+ */
+export interface RequestSqlParameterValue {
+    request: ConnectionParameterSource;
+}
+/**
+ * A lookup returning multiple rows when instantiated.
+ */
+export type ExpandingLookup = ParameterLookup | EvaluateTableValuedFunction;
+export interface ParameterLookup {
+    type: 'parameter';
+    lookup: StreamParameterIndexLookupCreator;
+    /**
+     * Must have the same length as {@link TableProcessor.parameters} for {@link lookup}.
+     */
+    instantiation: ParameterValue[];
+}
+export interface EvaluateTableValuedFunction {
+    type: 'table_valued';
+    functionName: string;
+    functionInputs: SqlExpression<RequestSqlParameterValue>[];
+    outputs: SqlExpression<ColumnSqlParameterValue>[];
+    filters: SqlExpression<ColumnSqlParameterValue>[];
+}
+export type ParameterValue = {
+    type: 'request';
+    expr: SqlExpression<RequestSqlParameterValue>;
+} | {
+    type: 'lookup';
+    lookup: ExpandingLookup;
+    resultIndex: number;
+} | {
+    type: 'intersection';
+    values: ParameterValue[];
+};

package/dist/sync_plan/plan.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=plan.js.map

package/dist/sync_plan/plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"plan.js","sourceRoot":"","sources":["../../src/sync_plan/plan.ts"],"names":[],"mappings":""}

package/dist/sync_plan/schema_inference.d.ts

@@ -0,0 +1,16 @@
+import { ColumnDefinition } from '../ExpressionType.js';
+import { SourceSchema } from '../types.js';
+import { StreamDataSource } from './plan.js';
+/**
+ * Infers the output schema of sync streams by resolving references against a statically-known source schema.
+ */
+export declare class SyncPlanSchemaAnalyzer {
+    private readonly defaultSchema;
+    private readonly schema;
+    constructor(defaultSchema: string, schema: SourceSchema);
+    /**
+     * Populates an output record of tables with the inferred result sets for a stream data source against a source
+     * schema.
+     */
+    resolveResultSets(source: StreamDataSource, tables: Record<string, Record<string, ColumnDefinition>>): void;
+}

package/dist/sync_plan/schema_inference.js

@@ -0,0 +1,123 @@
+import { CompatibilityContext } from '../compatibility.js';
+import { ExpressionType } from '../ExpressionType.js';
+import { generateSqlFunctions, getOperatorReturnType } from '../sql_functions.js';
+import { visitExpr } from './expression_visitor.js';
+/**
+ * Infers the output schema of sync streams by resolving references against a statically-known source schema.
+ */
+export class SyncPlanSchemaAnalyzer {
+    defaultSchema;
+    schema;
+    constructor(defaultSchema, schema) {
+        this.defaultSchema = defaultSchema;
+        this.schema = schema;
+    }
+    /**
+     * Populates an output record of tables with the inferred result sets for a stream data source against a source
+     * schema.
+     */
+    resolveResultSets(source, tables) {
+        for (const table of this.schema.getTables(source.sourceTable.toTablePattern(this.defaultSchema))) {
+            const typeResolver = new ExpressionTypeInference(table);
+            const outputName = source.outputTableName ?? table.name;
+            const outputTable = (tables[outputName] ??= {});
+            function addOutputColumn(definition) {
+                if (definition.name == 'id') {
+                    return; // Is implicit
+                }
+                const existing = outputTable[definition.name];
+                if (existing != null) {
+                    outputTable[definition.name] = {
+                        name: definition.name,
+                        type: existing.type.or(definition.type),
+                        originalType: definition.originalType == existing.originalType ? existing.originalType : undefined
+                    };
+                }
+                else {
+                    outputTable[definition.name] = definition;
+                }
+            }
+            for (const column of source.columns) {
+                if (column === 'star') {
+                    for (const actualColumn of table.getColumns()) {
+                        addOutputColumn(actualColumn);
+                    }
+                }
+                else {
+                    const type = visitExpr(typeResolver, column.expr, null);
+                    addOutputColumn({ name: column.alias, ...type });
+                }
+            }
+        }
+    }
+}
+/**
+ * Infers the type of expressions, resolving column references against a fixed schema table.
+ */
+class ExpressionTypeInference {
+    sourceTable;
+    constructor(sourceTable) {
+        this.sourceTable = sourceTable;
+    }
+    visitExternalData(expr) {
+        const column = this.sourceTable.getColumn(expr.source.column);
+        if (column) {
+            return { type: column.type, originalType: column.originalType };
+        }
+        return { type: ExpressionType.NONE };
+    }
+    visitUnaryExpression(expr) {
+        switch (expr.operator) {
+            case 'not':
+                return ExpressionTypeInference.BOOLEAN;
+            case '+':
+                return visitExpr(this, expr.operand, null);
+        }
+    }
+    visitBinaryExpression(expr) {
+        return {
+            type: getOperatorReturnType(expr.operator.toUpperCase(), visitExpr(this, expr.left, null).type, visitExpr(this, expr.right, null).type)
+        };
+    }
+    visitBetweenExpression() {
+        return ExpressionTypeInference.BOOLEAN;
+    }
+    visitScalarInExpression() {
+        return ExpressionTypeInference.BOOLEAN;
+    }
+    visitCaseWhenExpression(expr) {
+        let type = ExpressionType.NONE;
+        // Create a union of all THEN expressions (and ELSE, if present).
+        for (const { then } of expr.whens) {
+            type = type.or(visitExpr(this, then, null).type);
+        }
+        if (expr.else) {
+            type = type.or(visitExpr(this, expr.else, null).type);
+        }
+        return { type };
+    }
+    visitCastExpression(expr) {
+        return { type: ExpressionType.fromTypeText(expr.cast_as) };
+    }
+    visitScalarFunctionCallExpression(expr) {
+        const resolved = ExpressionTypeInference.functions.named[expr.function.toLowerCase()];
+        const args = expr.parameters.map((p) => visitExpr(this, p, null).type);
+        return { type: resolved.getReturnType(args) };
+    }
+    visitLiteralExpression(expr) {
+        switch (expr.type) {
+            case 'lit_null':
+                return { type: ExpressionType.NONE };
+            case 'lit_double':
+                return { type: ExpressionType.REAL };
+            case 'lit_int':
+                return { type: ExpressionType.INTEGER };
+            case 'lit_string':
+                return { type: ExpressionType.TEXT };
+        }
+    }
+    // We don't care about compatibility as these functions are only used to infer types.
+    static functions = generateSqlFunctions(CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY);
+    static BOOLEAN = { type: ExpressionType.INTEGER, originalType: 'bool' };
+}
+//# sourceMappingURL=schema_inference.js.map

package/dist/sync_plan/schema_inference.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema_inference.js","sourceRoot":"","sources":["../../src/sync_plan/schema_inference.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAgC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACpF,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAWlF,OAAO,EAAqB,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGvE;;GAEG;AACH,MAAM,OAAO,sBAAsB;IAEd;IACA;IAFnB,YACmB,aAAqB,EACrB,MAAoB;QADpB,kBAAa,GAAb,aAAa,CAAQ;QACrB,WAAM,GAAN,MAAM,CAAc;IACpC,CAAC;IAEJ;;;OAGG;IACH,iBAAiB,CAAC,MAAwB,EAAE,MAAwD;QAClG,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;YACjG,MAAM,YAAY,GAAG,IAAI,uBAAuB,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,IAAI,KAAK,CAAC,IAAI,CAAC;YACxD,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;YAEhD,SAAS,eAAe,CAAC,UAA4B;gBACnD,IAAI,UAAU,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;oBAC5B,OAAO,CAAC,cAAc;gBACxB,CAAC;gBAED,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG;wBAC7B,IAAI,EAAE,UAAU,CAAC,IAAI;wBACrB,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;wBACvC,YAAY,EAAE,UAAU,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;qBACnG,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;gBAC5C,CAAC;YACH,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;oBACtB,KAAK,MAAM,YAAY,IAAI,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC;wBAC9C,eAAe,CAAC,YAAY,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAG,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBACxD,eAAe,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,uBAAuB;IACE;IAA7B,YAA6B,WAA8B;QAA9B,gBAAW,GAAX,WAAW,CAAmB;IAAG,CAAC;IAE/D,iBAAiB,CAAC,IAA2C;QAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;QAClE,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,oBAAoB,CAAC,IAA8C;QACjE,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,KAAK,KAAK;gBACR,OAAO,uBAAuB,CAAC,OAAO,CAAC;YACzC,KAAK,GAAG;gBACN,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,qBAAqB,CAAC,IAA+C;QACnE,OAAO;YACL,IAAI,EAAE,qBAAqB,CACzB,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAC3B,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,EACrC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,CACvC;SACF,CAAC;IACJ,CAAC;IAED,sBAAsB;QACpB,OAAO,uBAAuB,CAAC,OAAO,CAAC;IACzC,CAAC;IAED,uBAAuB;QACrB,OAAO,uBAAuB,CAAC,OAAO,CAAC;IACzC,CAAC;IAED,uBAAuB,CAAC,IAAiD;QACvE,IAAI,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAC/B,iEAAiE;QACjE,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,CAAC;IAClB,CAAC;IAED,mBAAmB,CAAC,IAA6C;QAC/D,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,iCAAiC,CAAC,IAA2D;QAC3F,MAAM,QAAQ,GAAG,uBAAuB,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;IAChD,CAAC;IAED,sBAAsB,CAAC,IAAuB;QAC5C,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,KAAK,YAAY;gBACf,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,OAAO,EAAE,CAAC;YAC1C,KAAK,YAAY;gBACf,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,IAAI,EAAE,CAAC;QACzC,CAAC;IACH,CAAC;IAED,qFAAqF;IAC7E,MAAM,CAAU,SAAS,GAAG,oBAAoB,CAAC,oBAAoB,CAAC,4BAA4B,CAAC,CAAC;IACpG,MAAM,CAAU,OAAO,GAAe,EAAE,IAAI,EAAE,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC"}

package/dist/sync_plan/serialize.d.ts

@@ -0,0 +1,82 @@
+import { ParameterLookupScope } from '../HydrationState.js';
+import { SqlExpression } from './expression.js';
+import { ColumnSource, ColumnSqlParameterValue, PartitionKey, RequestSqlParameterValue, StreamOptions, SyncPlan } from './plan.js';
+/**
+ * Serializes a sync plan into a simple JSON object.
+ *
+ * While {@link SyncPlan}s are already serializable for the most part, it contains a graph of references from e.g.
+ * queriers to bucket creators. To represent this efficiently, we assign numbers to referenced elements while
+ * serializing instead of duplicating definitions.
+ */
+export declare function serializeSyncPlan(plan: SyncPlan): SerializedSyncPlanUnstable;
+export declare function deserializeSyncPlan(serialized: unknown): SyncPlan;
+interface SerializedSyncPlanUnstable {
+    version: 'unstable';
+    dataSources: SerializedDataSource[];
+    buckets: SerializedBucketDataSource[];
+    parameterIndexes: SerializedParameterIndexLookupCreator[];
+    streams: SerializedStream[];
+}
+interface SerializedBucketDataSource {
+    hash: number;
+    uniqueName: string;
+    sources: number[];
+}
+interface SerializedTablePattern {
+    connection: string | null;
+    schema: string | null;
+    table: string;
+}
+interface SerializedDataSource {
+    table: SerializedTablePattern;
+    outputTableName?: string;
+    hash: number;
+    columns: ColumnSource[];
+    filters: SqlExpression<ColumnSqlParameterValue>[];
+    partitionBy: PartitionKey[];
+}
+interface SerializedParameterIndexLookupCreator {
+    table: SerializedTablePattern;
+    hash: number;
+    lookupScope: ParameterLookupScope;
+    output: SqlExpression<ColumnSqlParameterValue>[];
+    filters: SqlExpression<ColumnSqlParameterValue>[];
+    partitionBy: PartitionKey[];
+}
+interface SerializedStream {
+    stream: StreamOptions;
+    queriers: SerializedStreamQuerier[];
+}
+interface SerializedStreamQuerier {
+    requestFilters: SqlExpression<RequestSqlParameterValue>[];
+    lookupStages: SerializedExpandingLookup[][];
+    bucket: number;
+    sourceInstantiation: SerializedParameterValue[];
+}
+type SerializedExpandingLookup = {
+    type: 'parameter';
+    lookup: number;
+    instantiation: SerializedParameterValue[];
+} | {
+    type: 'table_valued';
+    functionName: string;
+    functionInputs: SqlExpression<RequestSqlParameterValue>[];
+    outputs: SqlExpression<ColumnSqlParameterValue>[];
+    filters: SqlExpression<ColumnSqlParameterValue>[];
+};
+interface LookupReference {
+    stageId: number;
+    idInStage: number;
+}
+type SerializedParameterValue = {
+    type: 'request';
+    expr: SqlExpression<RequestSqlParameterValue>;
+} | {
+    type: 'lookup';
+    lookup: LookupReference;
+    resultIndex: number;
+} | {
+    type: 'intersection';
+    values: SerializedParameterValue[];
+};
+export {};

package/dist/sync_plan/serialize.js

@@ -0,0 +1,214 @@
+import { ImplicitSchemaTablePattern, TablePattern } from '../TablePattern.js';
+/**
+ * Serializes a sync plan into a simple JSON object.
+ *
+ * While {@link SyncPlan}s are already serializable for the most part, it contains a graph of references from e.g.
+ * queriers to bucket creators. To represent this efficiently, we assign numbers to referenced elements while
+ * serializing instead of duplicating definitions.
+ */
+export function serializeSyncPlan(plan) {
+    const dataSourceIndex = new Map();
+    const bucketIndex = new Map();
+    const parameterIndex = new Map();
+    const expandingLookups = new Map();
+    function serializeTablePattern(pattern) {
+        return {
+            connection: pattern.connectionTag,
+            schema: pattern.schema,
+            table: pattern.tablePattern
+        };
+    }
+    function serializeDataSources() {
+        return plan.dataSources.map((source, i) => {
+            dataSourceIndex.set(source, i);
+            return {
+                hash: source.hashCode,
+                table: serializeTablePattern(source.sourceTable),
+                outputTableName: source.outputTableName,
+                filters: source.filters,
+                partitionBy: source.parameters,
+                columns: source.columns
+            };
+        });
+    }
+    function serializeParameterIndexes() {
+        return plan.parameterIndexes.map((source, i) => {
+            parameterIndex.set(source, i);
+            return {
+                hash: source.hashCode,
+                table: serializeTablePattern(source.sourceTable),
+                filters: source.filters,
+                partitionBy: source.parameters,
+                output: source.outputs,
+                lookupScope: source.defaultLookupScope
+            };
+        });
+    }
+    function serializeParameterValue(value) {
+        if (value.type == 'request') {
+            return { type: 'request', expr: value.expr };
+        }
+        else if (value.type == 'lookup') {
+            return { type: 'lookup', lookup: expandingLookups.get(value.lookup), resultIndex: value.resultIndex };
+        }
+        else {
+            return { type: 'intersection', values: value.values.map(serializeParameterValue) };
+        }
+    }
+    function serializeStreamQuerier(source) {
+        const stages = [];
+        source.lookupStages.map((stage, stageIndex) => {
+            stages.push(stage.map((e, indexInStage) => {
+                const ref = {
+                    stageId: stageIndex,
+                    idInStage: indexInStage
+                };
+                let mapped;
+                if (e.type == 'parameter') {
+                    mapped = {
+                        type: 'parameter',
+                        lookup: parameterIndex.get(e.lookup),
+                        instantiation: e.instantiation.map(serializeParameterValue)
+                    };
+                }
+                else {
+                    mapped = {
+                        type: 'table_valued',
+                        functionName: e.functionName,
+                        functionInputs: e.functionInputs,
+                        outputs: e.outputs,
+                        filters: e.filters
+                    };
+                }
+                expandingLookups.set(e, ref);
+                return mapped;
+            }));
+        });
+        return {
+            requestFilters: source.requestFilters,
+            lookupStages: stages,
+            bucket: bucketIndex.get(source.bucket),
+            sourceInstantiation: source.sourceInstantiation.map(serializeParameterValue)
+        };
+    }
+    return {
+        version: 'unstable', // TODO: Mature to 1 before storing in bucket storage
+        dataSources: serializeDataSources(),
+        buckets: plan.buckets.map((bkt, index) => {
+            bucketIndex.set(bkt, index);
+            return {
+                hash: bkt.hashCode,
+                uniqueName: bkt.uniqueName,
+                sources: bkt.sources.map((e) => dataSourceIndex.get(e))
+            };
+        }),
+        parameterIndexes: serializeParameterIndexes(),
+        streams: plan.streams.map((s) => ({
+            stream: s.stream,
+            queriers: s.queriers.map(serializeStreamQuerier)
+        }))
+    };
+}
+export function deserializeSyncPlan(serialized) {
+    // TODO: Mature to version 1
+    if (serialized.version != 'unstable') {
+        throw new Error('Unknown sync plan version passed to deserializeSyncPlan()');
+    }
+    function deserializeTablePattern(pattern) {
+        if (pattern.schema) {
+            return new TablePattern(`${pattern.connection}.${pattern.schema}`, pattern.table);
+        }
+        else {
+            return new ImplicitSchemaTablePattern(null, pattern.table);
+        }
+    }
+    const plan = serialized;
+    const dataSources = plan.dataSources.map((source) => {
+        return {
+            hashCode: source.hash,
+            sourceTable: deserializeTablePattern(source.table),
+            outputTableName: source.outputTableName,
+            filters: source.filters,
+            parameters: source.partitionBy,
+            columns: source.columns
+        };
+    });
+    const buckets = plan.buckets.map((bkt) => {
+        return {
+            hashCode: bkt.hash,
+            uniqueName: bkt.uniqueName,
+            sources: bkt.sources.map((idx) => dataSources[idx])
+        };
+    });
+    const parameterIndexes = plan.parameterIndexes.map((source) => {
+        return {
+            hashCode: source.hash,
+            sourceTable: deserializeTablePattern(source.table),
+            filters: source.filters,
+            parameters: source.partitionBy,
+            outputs: source.output,
+            defaultLookupScope: source.lookupScope
+        };
+    });
+    function deserializeParameterValue(stages, value) {
+        switch (value.type) {
+            case 'request':
+                return value;
+            case 'lookup':
+                return {
+                    type: 'lookup',
+                    lookup: stages[value.lookup.stageId][value.lookup.idInStage],
+                    resultIndex: value.resultIndex
+                };
+            case 'intersection':
+                return { type: 'intersection', values: value.values.map((v) => deserializeParameterValue(stages, v)) };
+        }
+    }
+    function deserializeExpandingLookup(stages, source) {
+        switch (source.type) {
+            case 'parameter':
+                return {
+                    type: 'parameter',
+                    lookup: parameterIndexes[source.lookup],
+                    instantiation: source.instantiation.map((v) => deserializeParameterValue(stages, v))
+                };
+            case 'table_valued':
+                return {
+                    type: 'table_valued',
+                    functionName: source.functionName,
+                    functionInputs: source.functionInputs,
+                    outputs: source.outputs,
+                    filters: source.filters
+                };
+        }
+    }
+    function deserializeStreamQuerier(source) {
+        const lookupStages = [];
+        for (const serializedStage of source.lookupStages) {
+            const stage = [];
+            for (const serializedElement of serializedStage) {
+                stage.push(deserializeExpandingLookup(lookupStages, serializedElement));
+            }
+            lookupStages.push(stage);
+        }
+        return {
+            requestFilters: source.requestFilters,
+            lookupStages,
+            bucket: buckets[source.bucket],
+            sourceInstantiation: source.sourceInstantiation.map((v) => deserializeParameterValue(lookupStages, v))
+        };
+    }
+    const streams = plan.streams.map((source) => {
+        return {
+            stream: source.stream,
+            queriers: source.queriers.map(deserializeStreamQuerier)
+        };
+    });
+    return {
+        dataSources,
+        buckets,
+        parameterIndexes,
+        streams
+    };
+}
+//# sourceMappingURL=serialize.js.map

package/dist/sync_plan/serialize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialize.js","sourceRoot":"","sources":["../../src/sync_plan/serialize.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAoB9E;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,MAAM,eAAe,GAAG,IAAI,GAAG,EAA4B,CAAC;IAC5D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkC,CAAC;IAC9D,MAAM,cAAc,GAAG,IAAI,GAAG,EAA6C,CAAC;IAC5E,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;IAErE,SAAS,qBAAqB,CAAC,OAAmC;QAChE,OAAO;YACL,UAAU,EAAE,OAAO,CAAC,aAAa;YACjC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,YAAY;SAC5B,CAAC;IACJ,CAAC;IAED,SAAS,oBAAoB;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACxC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAE/B,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,QAAQ;gBACrB,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAChD,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,WAAW,EAAE,MAAM,CAAC,UAAU;gBAC9B,OAAO,EAAE,MAAM,CAAC,OAAO;aACO,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,yBAAyB;QAChC,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAE9B,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,QAAQ;gBACrB,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAChD,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,WAAW,EAAE,MAAM,CAAC,UAAU;gBAC9B,MAAM,EAAE,MAAM,CAAC,OAAO;gBACtB,WAAW,EAAE,MAAM,CAAC,kBAAkB;aACS,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,uBAAuB,CAAC,KAAqB;QACpD,IAAI,KAAK,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QACzG,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,EAAE,CAAC;QACrF,CAAC;IACH,CAAC;IAED,SAAS,sBAAsB,CAAC,MAAqB;QACnD,MAAM,MAAM,GAAkC,EAAE,CAAC;QAEjD,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAC5C,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE;gBAC5B,MAAM,GAAG,GAAoB;oBAC3B,OAAO,EAAE,UAAU;oBACnB,SAAS,EAAE,YAAY;iBACxB,CAAC;gBACF,IAAI,MAAiC,CAAC;gBAEtC,IAAI,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC;oBAC1B,MAAM,GAAG;wBACP,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAE;wBACrC,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,uBAAuB,CAAC;qBAC5D,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG;wBACP,IAAI,EAAE,cAAc;wBACpB,YAAY,EAAE,CAAC,CAAC,YAAY;wBAC5B,cAAc,EAAE,CAAC,CAAC,cAAc;wBAChC,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;qBACnB,CAAC;gBACJ,CAAC;gBAED,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC7B,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,YAAY,EAAE,MAAM;YACpB,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAE;YACvC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,uBAAuB,CAAC;SAC7E,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,UAAU,EAAE,qDAAqD;QAC1E,WAAW,EAAE,oBAAoB,EAAE;QACnC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACvC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5B,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,QAAQ;gBAClB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;aACzD,CAAC;QACJ,CAAC,CAAC;QACF,gBAAgB,EAAE,yBAAyB,EAAE;QAC7C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,sBAAsB,CAAC;SACjD,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAmB;IACrD,4BAA4B;IAC5B,IAAK,UAAyC,CAAC,OAAO,IAAI,UAAU,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,SAAS,uBAAuB,CAAC,OAA+B;QAC9D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,IAAI,YAAY,CAAC,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACpF,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,0BAA0B,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,UAAwC,CAAC;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAClD,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,IAAI;YACrB,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,KAAK,CAAC;YAClD,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,OAAO,EAAE,MAAM,CAAC,OAAO;SACG,CAAC;IAC/B,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACvC,OAAO;YACL,QAAQ,EAAE,GAAG,CAAC,IAAI;YAClB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;SACnB,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5D,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,IAAI;YACrB,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,KAAK,CAAC;YAClD,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,OAAO,EAAE,MAAM,CAAC,MAAM;YACtB,kBAAkB,EAAE,MAAM,CAAC,WAAW;SACK,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,SAAS,yBAAyB,CAAC,MAA2B,EAAE,KAA+B;QAC7F,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,SAAS;gBACZ,OAAO,KAAK,CAAC;YACf,KAAK,QAAQ;gBACX,OAAO;oBACL,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC;oBAC5D,WAAW,EAAE,KAAK,CAAC,WAAW;iBAC/B,CAAC;YACJ,KAAK,cAAc;gBACjB,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,SAAS,0BAA0B,CAAC,MAA2B,EAAE,MAAiC;QAChG,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,WAAW;gBACd,OAAO;oBACL,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC;oBACvC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;iBAC3D,CAAC;YAC9B,KAAK,cAAc;gBACjB,OAAO;oBACL,IAAI,EAAE,cAAc;oBACpB,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,cAAc,EAAE,MAAM,CAAC,cAAc;oBACrC,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;iBACc,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,SAAS,wBAAwB,CAAC,MAA+B;QAC/D,MAAM,YAAY,GAAwB,EAAE,CAAC;QAC7C,KAAK,MAAM,eAAe,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAClD,MAAM,KAAK,GAAsB,EAAE,CAAC;YACpC,KAAK,MAAM,iBAAiB,IAAI,eAAe,EAAE,CAAC;gBAChD,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC,CAAC;YAC1E,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO;YACL,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,YAAY;YACZ,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9B,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;SACvG,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC1C,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,wBAAwB,CAAC;SAC3B,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,WAAW;QACX,OAAO;QACP,gBAAgB;QAChB,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/types/custom_sqlite_value.d.ts

@@ -1,6 +1,6 @@
 import { CompatibilityContext } from '../compatibility.js';
-import { SqliteValue } from '../types.js';
 import { SqliteValueType } from '../ExpressionType.js';
+import { SqliteValue } from '../types.js';
 /**
  * A value that decays into a {@link SqliteValue} in a context-specific way.
  *