@sixb/sqlite 0.1.0 → 0.1.1

package/dist/index.js

@@ -1108,6 +1108,110 @@ CREATE INDEX idx_agent_messages_project_thread_role
   ON agent_messages(project_id, thread_id, role, seq);
 `;
 
+// src/migrations/002-workflow-run-output.sql
+var _002_workflow_run_output_default = `ALTER TABLE workflow_runs ADD COLUMN output TEXT;
+
+UPDATE workflow_runs AS run
+SET output = CASE
+  WHEN EXISTS (
+    SELECT 1
+    FROM workflow_node_runs AS node
+    WHERE node.project_id = run.project_id
+      AND node.workflow_run_id = run.id
+      AND node.status = 'succeeded'
+      AND node.node_type <> 'action'
+  ) THEN COALESCE(
+    (
+      SELECT node.output
+      FROM workflow_node_runs AS node
+      WHERE node.project_id = run.project_id
+        AND node.workflow_run_id = run.id
+        AND node.status = 'succeeded'
+        AND node.node_type <> 'action'
+      ORDER BY node.node_index DESC, node.id DESC
+      LIMIT 1
+    ),
+    '{}'
+  )
+  ELSE run.input
+END
+WHERE run.status = 'succeeded';
+`;
+
+// src/migrations/003-merge-sync-runs.sql
+var _003_merge_sync_runs_default = `DROP INDEX idx_sync_runs_project_started;
+DROP INDEX idx_sync_runs_project_dataset_started;
+DROP INDEX idx_sync_runs_project_sync_started;
+DROP INDEX idx_sync_runs_project_status_started;
+
+ALTER TABLE sync_runs RENAME TO sync_runs_before_merge_mode;
+
+CREATE TABLE sync_runs (
+  project_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  sync_id TEXT NOT NULL,
+  dataset_id TEXT NOT NULL,
+  mode TEXT NOT NULL CHECK (mode IN ('snapshot', 'append', 'merge')),
+  status TEXT NOT NULL CHECK (status IN ('running', 'succeeded', 'failed', 'cancelled')),
+  started_at TEXT NOT NULL,
+  finished_at TEXT,
+  rows_read INTEGER CHECK (rows_read IS NULL OR rows_read >= 0),
+  output_version_id TEXT,
+  expected_latest_version_id TEXT,
+  commit_message TEXT,
+  error_name TEXT,
+  error_message TEXT,
+  checkpoint TEXT,
+  PRIMARY KEY (project_id, id)
+);
+
+INSERT INTO sync_runs (
+  project_id,
+  id,
+  sync_id,
+  dataset_id,
+  mode,
+  status,
+  started_at,
+  finished_at,
+  rows_read,
+  output_version_id,
+  expected_latest_version_id,
+  commit_message,
+  error_name,
+  error_message,
+  checkpoint
+)
+SELECT
+  project_id,
+  id,
+  sync_id,
+  dataset_id,
+  mode,
+  status,
+  started_at,
+  finished_at,
+  rows_read,
+  output_version_id,
+  expected_latest_version_id,
+  commit_message,
+  error_name,
+  error_message,
+  checkpoint
+FROM sync_runs_before_merge_mode;
+
+DROP TABLE sync_runs_before_merge_mode;
+
+CREATE INDEX idx_sync_runs_project_started
+  ON sync_runs(project_id, started_at DESC);
+CREATE INDEX idx_sync_runs_project_dataset_started
+  ON sync_runs(project_id, dataset_id, started_at DESC);
+CREATE INDEX idx_sync_runs_project_sync_started
+  ON sync_runs(project_id, sync_id, started_at DESC);
+CREATE INDEX idx_sync_runs_project_status_started
+  ON sync_runs(project_id, status, started_at DESC);
+`;
+
 // src/migrations.ts
 var MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -1125,7 +1229,11 @@ var SQLITE_STORAGE_ADAPTER_ID = "SixbSqliteStorage";
 var SQLITE_STORAGE_FILE = "storage.sqlite";
 var sqliteStorageMigrations = defineMigrations({
   adapterId: SQLITE_STORAGE_ADAPTER_ID,
-  steps: [sqliteSql("001-initial-schema", _001_initial_schema_default)]
+  steps: [
+    sqliteSql("001-initial-schema", _001_initial_schema_default),
+    sqliteSql("002-workflow-run-output", _002_workflow_run_output_default),
+    sqliteSql("003-merge-sync-runs", _003_merge_sync_runs_default)
+  ]
 });
 function sqliteStoragePath(basePath) {
   return `${basePath}/${SQLITE_STORAGE_FILE}`;
@@ -5722,22 +5830,30 @@ class SqliteMaterializationStateReader {
       FROM json_each(?)
     `;
     const effectiveRows = this.db.query(`WITH requested AS (${requestedObjects})
-         SELECT objects.* FROM objects
-         JOIN requested USING (object_type_id, primary_id)
-         WHERE objects.project_id = ?`).all(requested, this.projectId);
+         SELECT objects.* FROM requested
+         CROSS JOIN objects
+           ON objects.project_id = ?
+          AND objects.object_type_id = requested.object_type_id
+          AND objects.primary_id = requested.primary_id`).all(requested, this.projectId);
     const sourceRows = this.db.query(`WITH requested AS (${requestedObjects})
-         SELECT rows.* FROM ontology_source_rows AS rows
-         JOIN requested USING (object_type_id, primary_id)
+         SELECT rows.* FROM requested
+         CROSS JOIN ontology_source_rows AS rows
+           ON rows.project_id = ?
+          AND rows.entity_kind = 'object'
+          AND rows.object_type_id = requested.object_type_id
+          AND rows.primary_id = requested.primary_id
          JOIN ontology_sources AS sources
            ON sources.project_id = rows.project_id
           AND sources.source_id = rows.source_id
           AND sources.materialization_id = rows.materialization_id
-         WHERE rows.project_id = ? AND rows.entity_kind = 'object'
-           AND sources.status = 'active'`).all(requested, this.projectId);
+         WHERE sources.status = 'active'`).all(requested, this.projectId);
     const overrideRows = this.db.query(`WITH requested AS (${requestedObjects})
-         SELECT overrides.* FROM ontology_overrides AS overrides
-         JOIN requested USING (object_type_id, primary_id)
-         WHERE overrides.project_id = ? AND overrides.entity_kind = 'object'`).all(requested, this.projectId);
+         SELECT overrides.* FROM requested
+         CROSS JOIN ontology_overrides AS overrides
+           ON overrides.project_id = ?
+          AND overrides.entity_kind = 'object'
+          AND overrides.object_type_id = requested.object_type_id
+          AND overrides.primary_id = requested.primary_id`).all(requested, this.projectId);
     const telemetryRows = this.db.query(`WITH requested AS (${requestedObjects}), ranked AS (
            SELECT timeseries.*,
              ROW_NUMBER() OVER (
@@ -5745,11 +5861,11 @@ class SqliteMaterializationStateReader {
                  timeseries.property_id
                ORDER BY timeseries.at DESC
              ) AS rank
-           FROM timeseries
-           JOIN requested
-             ON requested.object_type_id = timeseries.object_type_id
-            AND requested.primary_id = timeseries.object_id
-           WHERE timeseries.project_id = ?
+           FROM requested
+           CROSS JOIN timeseries
+             ON timeseries.project_id = ?
+            AND timeseries.object_type_id = requested.object_type_id
+            AND timeseries.object_id = requested.primary_id
          )
          SELECT * FROM ranked WHERE rank = 1`).all(requested, this.projectId);
     const effective = new Map(effectiveRows.map((row) => [objectRefKey(objectRefFromColumns(row)), row]));
@@ -5807,26 +5923,39 @@ class SqliteMaterializationStateReader {
       FROM json_each(?)
     `;
     const effectiveRows = this.db.query(`WITH requested AS (${effectiveRequest})
-         SELECT links.* FROM links
-         JOIN requested USING (source_type_id, source_id, link_id, target_type_id, target_id)
-         WHERE links.project_id = ?`).all(requested, this.projectId);
+         SELECT links.* FROM requested
+         CROSS JOIN links
+           ON links.project_id = ?
+          AND links.source_type_id = requested.source_type_id
+          AND links.source_id = requested.source_id
+          AND links.link_id = requested.link_id
+          AND links.target_type_id = requested.target_type_id
+          AND links.target_id = requested.target_id`).all(requested, this.projectId);
     const sourceRows = this.db.query(`WITH requested AS (${sourceRequest})
-         SELECT rows.* FROM ontology_source_rows AS rows
-         JOIN requested USING (
-           source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
-         )
+         SELECT rows.* FROM requested
+         CROSS JOIN ontology_source_rows AS rows
+           ON rows.project_id = ?
+          AND rows.entity_kind = 'link'
+          AND rows.source_type_id = requested.source_type_id
+          AND rows.source_primary_id = requested.source_primary_id
+          AND rows.link_id = requested.link_id
+          AND rows.target_type_id = requested.target_type_id
+          AND rows.target_primary_id = requested.target_primary_id
          JOIN ontology_sources AS sources
            ON sources.project_id = rows.project_id
           AND sources.source_id = rows.source_id
           AND sources.materialization_id = rows.materialization_id
-         WHERE rows.project_id = ? AND rows.entity_kind = 'link'
-           AND sources.status = 'active'`).all(requested, this.projectId);
+         WHERE sources.status = 'active'`).all(requested, this.projectId);
     const overrideRows = this.db.query(`WITH requested AS (${sourceRequest})
-         SELECT overrides.* FROM ontology_overrides AS overrides
-         JOIN requested USING (
-           source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
-         )
-         WHERE overrides.project_id = ? AND overrides.entity_kind = 'link'`).all(requested, this.projectId);
+         SELECT overrides.* FROM requested
+         CROSS JOIN ontology_overrides AS overrides
+           ON overrides.project_id = ?
+          AND overrides.entity_kind = 'link'
+          AND overrides.source_type_id = requested.source_type_id
+          AND overrides.source_primary_id = requested.source_primary_id
+          AND overrides.link_id = requested.link_id
+          AND overrides.target_type_id = requested.target_type_id
+          AND overrides.target_primary_id = requested.target_primary_id`).all(requested, this.projectId);
     const effective = new Map(effectiveRows.map((row) => [linkRefKey(linkRefFromColumns(row)), row]));
     const sources = activeSourceMap(sourceRows);
     const overrides = new Map(overrideRows.map((row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)]));
@@ -5860,9 +5989,13 @@ class SqliteMaterializationStateReader {
              json_extract(value, '$.at') AS at
            FROM json_each(?)
          )
-         SELECT timeseries.* FROM timeseries
-         JOIN requested USING (object_type_id, object_id, property_id, at)
-         WHERE timeseries.project_id = ?`).all(requested, this.projectId);
+         SELECT timeseries.* FROM requested
+         CROSS JOIN timeseries
+           ON timeseries.project_id = ?
+          AND timeseries.object_type_id = requested.object_type_id
+          AND timeseries.object_id = requested.object_id
+          AND timeseries.property_id = requested.property_id
+          AND timeseries.at = requested.at`).all(requested, this.projectId);
     const found = new Map(rows.map((row) => {
       const point = storedPoint(row);
       return [telemetryPointSortKey(point.series, point.at), point];
@@ -5895,9 +6028,12 @@ class SqliteMaterializationStateReader {
             FROM json_each(?)
           )
           SELECT requested.scope_sort_key, links.*
-          FROM links
-          JOIN requested USING (source_type_id, source_id, link_id)
-          WHERE links.project_id = ?
+          FROM requested
+          CROSS JOIN links
+            ON links.project_id = ?
+           AND links.source_type_id = requested.source_type_id
+           AND links.source_id = requested.source_id
+           AND links.link_id = requested.link_id
           ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
         `).iterate(canonicalJson(requested), this.projectId);
     for (const row of rows) {
@@ -6041,17 +6177,23 @@ class SqliteMaterializationStateReader {
     if (materializationIds.length === 0 || refs.length === 0) {
       return { owned: new Set, byMaterialization: new Map };
     }
-    const requestedKeys = canonicalJson(refs.map((ref) => projectionEntityKey(ref)));
+    const requestedEntities = canonicalJson(refs.map((ref) => ({ entityKind: ref.kind, entityKey: projectionEntityKey(ref) })));
     const requestedMaterializations = canonicalJson([...new Set(materializationIds)]);
     const rows = this.db.query(`WITH requested_entities AS (
-           SELECT value AS entity_key FROM json_each(?)
+           SELECT DISTINCT json_extract(value, '$.entityKind') AS entity_kind,
+             json_extract(value, '$.entityKey') AS entity_key
+           FROM json_each(?)
          ), requested_materializations AS (
            SELECT value AS materialization_id FROM json_each(?)
          )
-         SELECT rows.* FROM ontology_source_rows AS rows
-         JOIN requested_entities USING (entity_key)
-         JOIN requested_materializations USING (materialization_id)
-         WHERE rows.project_id = ? AND rows.source_id = ?`).all(requestedKeys, requestedMaterializations, this.projectId, sourceId);
+         SELECT rows.* FROM requested_materializations
+         CROSS JOIN requested_entities
+         CROSS JOIN ontology_source_rows AS rows
+           ON rows.project_id = ?
+          AND rows.source_id = ?
+          AND rows.materialization_id = requested_materializations.materialization_id
+          AND rows.entity_kind = requested_entities.entity_kind
+          AND rows.entity_key = requested_entities.entity_key`).all(requestedEntities, requestedMaterializations, this.projectId, sourceId);
     const owned = new Set;
     const byMaterialization = new Map;
     for (const row of rows) {
@@ -8670,8 +8812,8 @@ class SqliteSyncRunStorage {
         if (input.output && input.output.datasetId !== existing.dataset_id) {
           throw new SyncRunError(`[SixbSqlite] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.dataset_id}'.`);
         }
-        if (!input.output && (existing.mode !== "append" || input.rowsRead !== 0)) {
-          throw new SyncRunError(`[SixbSqlite] Sync run '${input.id}' may omit its output only for an empty append.`);
+        if (!input.output && input.rowsRead !== 0 && existing.mode !== "merge") {
+          throw new SyncRunError(`[SixbSqlite] Sync run '${input.id}' may omit its output with rows read only for an initial merge no-op.`);
         }
       }
       this.db.query(`
@@ -9541,10 +9683,12 @@ class SqliteWorkflowRunStorage {
           SET
             status = ?,
             finished_at = ?,
-            error = ?
-            , execution_token = NULL, execution_queue_lease_expires_at = NULL
+            output = ?,
+            error = ?,
+            execution_token = NULL,
+            execution_queue_lease_expires_at = NULL
           WHERE project_id = ? AND id = ?
-        `).run(input.status, (input.finishedAt ?? new Date).toISOString(), input.status === "succeeded" ? null : input.error ?? null, input.projectId, input.id);
+        `).run(input.status, (input.finishedAt ?? new Date).toISOString(), input.status === "succeeded" ? serializeRecord2(input.output) : null, input.status === "succeeded" ? null : input.error ?? null, input.projectId, input.id);
       const updated = this.db.query("SELECT * FROM workflow_runs WHERE project_id = ? AND id = ?").get(input.projectId, input.id);
       return rowToWorkflowRunRecord(updated);
     })();
@@ -9954,6 +10098,7 @@ function rowToWorkflowRunRecord(row) {
     workflowId: row.workflow_id,
     status: row.status,
     input: parseRecord2(row.input),
+    output: row.output ? parseRecord2(row.output) : undefined,
     queuedAt: row.queued_at ? new Date(row.queued_at) : undefined,
     startedAt: new Date(row.started_at),
     finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
@@ -10201,4 +10346,4 @@ export {
   SqliteStorage
 };
 
-//# debugId=983F7BC9E5B8E71164756E2164756E21
+//# debugId=531479881454EA0264756E2164756E21