@powerhousedao/reactor-workflow 6.2.3-dev.33 → 6.2.3-dev.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
- import { H as authMethodFor, S as maxFileBytes, U as unsupportedAuth, V as UnsupportedPieceFeatureError, W as unsupportedTrigger, a as redactMessage, c as DEFAULT_EGRESS_POLICY, d as parseAddress, m as extractDedupeKey, n as containsRedactedMarker, o as rememberSecrets, r as redact, s as secretsFor, u as isPrivateAddress, y as FileTooLargeError } from "./redact-LRWHJGW9.js";
2
- import { A as expressionsIn, B as PieceWorker, C as describeBuiltinPiece, E as BoundConnectionResolver, F as parseSecretRef, G as sourcedResolver, H as PieceWorkerTimeoutError, I as secretRefFromId, J as npmRegistryBaseUrl, L as stepBlock, N as SecretDeletedError, O as declaredConnectionIds, P as SecretNotFoundError, R as triggerBlock, S as builtinPiece, T as CORE_PIECE_VERSION, U as bundleResolver, V as PieceWorkerError, W as pieceModuleRef, X as setPieceRegistryUrl, Y as pieceRegistrySource, _ as servesReactorPort, a as runWorkflow, b as unpinnedNote, c as stepConfigHash, d as ActivepiecesBlockExecutor, f as CompositeBlockExecutor, g as resolvedBlock, h as reactorHandlers, i as packagePieces, j as resolveExpressions, k as shapeConnection, l as DynamicPropertiesError, m as isHostBound, n as installedPiece, o as canonicalJson, p as REACTOR_PORT_PIECE, r as installedPieces, s as hashOf, t as PieceRegistry, u as checkDynamicProperties, v as storeHandlers, w as CORE_PIECE_NAME, x as blockPorts, y as blockLabel, z as PieceWorkerPool } from "./piece-registry-Cy1a1UIa.js";
1
+ import { G as unsupportedTrigger, H as UnsupportedPieceFeatureError, S as maxFileBytes, U as authMethodFor, W as unsupportedAuth, a as redactMessage, c as DEFAULT_EGRESS_POLICY, d as parseAddress, m as extractDedupeKey, n as containsRedactedMarker, o as rememberSecrets, r as redact, s as secretsFor, u as isPrivateAddress, y as FileTooLargeError } from "./redact-DW7W5rrV.js";
2
+ import { A as resolveStepInput, B as PieceWorker, C as describeBuiltinPiece, E as BoundConnectionResolver, F as parseSecretRef, G as sourcedResolver, H as PieceWorkerTimeoutError, I as secretRefFromId, J as npmRegistryBaseUrl, L as stepBlock, N as SecretDeletedError, O as declaredConnectionIds, P as SecretNotFoundError, R as triggerBlock, S as builtinPiece, T as CORE_PIECE_VERSION, U as bundleResolver, V as PieceWorkerError, W as pieceModuleRef, X as setPieceRegistryUrl, Y as pieceRegistrySource, _ as servesReactorPort, a as runWorkflow, b as unpinnedNote, c as stepConfigHash, d as ActivepiecesBlockExecutor, f as CompositeBlockExecutor, g as resolvedBlock, h as reactorHandlers, i as packagePieces, j as expressionsIn, k as shapeConnection, l as DynamicPropertiesError, m as isHostBound, n as installedPiece, o as canonicalJson, p as REACTOR_PORT_PIECE, r as installedPieces, s as hashOf, t as PieceRegistry, u as checkDynamicProperties, v as storeHandlers, w as CORE_PIECE_NAME, x as blockPorts, y as blockLabel, z as PieceWorkerPool } from "./piece-registry-BBLaiiwi.js";
3
3
  import { WebhookHandshakeStrategy } from "@powerhousedao/pieces-framework";
4
4
  import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
5
5
  import { open, readFile, rm } from "node:fs/promises";
@@ -17,10 +17,11 @@ import http from "node:http";
17
17
  import https from "node:https";
18
18
  import { actions as actions$1 } from "@powerhousedao/workflow/document-models/workflow";
19
19
  import { Kind, parse } from "graphql";
20
- import { BaseReadModel, defaultReadModelIndexingConfig } from "@powerhousedao/reactor";
20
+ import { BaseReadModel, JobStatus, defaultReadModelIndexingConfig, isPurgeMarker } from "@powerhousedao/reactor";
21
+ import { addFile } from "@powerhousedao/shared/document-drive";
21
22
  //#region src/pieces/activepieces/piece-versions.ts
22
23
  const LISTED_TTL_MS = 3e5;
23
- const ABSENT_TTL_MS = 3e5;
24
+ const ABSENT_TTL_MS = 3e4;
24
25
  const DEFAULT_TIMEOUT_MS = 5e3;
25
26
  const cache = /* @__PURE__ */ new Map();
26
27
  const inFlight = /* @__PURE__ */ new Map();
@@ -131,7 +132,7 @@ function createAttachmentPort(client, documentIdFor, canReadRef) {
131
132
  async read(ref, destPath) {
132
133
  const documentId = documentIdFor();
133
134
  if (!documentId) throw new Error(`Cannot resolve ${ref}: no workflow document is in scope to authorize the read`);
134
- if (!await canReadRef(documentId, ref)) throw new Error(`Cannot resolve ${ref}: workflow document "${documentId}" does not reference it`);
135
+ if (!await canReadRef(documentId, ref)) throw new Error(`Cannot resolve ${ref}: workflow "${documentId}" may not read it`);
135
136
  const limit = maxFileBytes();
136
137
  const { header, body } = await client.download({
137
138
  documentId,
@@ -754,7 +755,10 @@ const SERVER_ONLY_PIECES = /* @__PURE__ */ new Set([
754
755
  //#endregion
755
756
  //#region src/reactor/piece-catalog.ts
756
757
  const CATALOG_URL = "https://cloud.activepieces.com/api/v1/pieces";
757
- const CACHE_TTL_MS = 36e5;
758
+ const CACHE_TTL_MS = 3e5;
759
+ const PINNED_TTL_MS = 36e5;
760
+ const REGISTRY_PAGE_SIZE = 50;
761
+ const REGISTRY_MAX_PAGES = 200;
758
762
  const logger$5 = childLogger(["workflow", "piece-catalog"]);
759
763
  function aiLast(audience) {
760
764
  return audience === "ai" ? 1 : 0;
@@ -790,6 +794,20 @@ async function fetchJson(url, timeoutMs = 3e4) {
790
794
  if (!response.ok) throw new CatalogStatusError(url, response.status);
791
795
  return response.json();
792
796
  }
797
+ async function fetchRegistryCatalog(url, timeoutMs) {
798
+ const items = [];
799
+ for (let n = 0; n < REGISTRY_MAX_PAGES; n++) {
800
+ const pageUrl = new URL(url);
801
+ pageUrl.searchParams.set("limit", String(REGISTRY_PAGE_SIZE));
802
+ pageUrl.searchParams.set("offset", String(items.length));
803
+ const page = await fetchJson(pageUrl.toString(), timeoutMs);
804
+ if (!Array.isArray(page.items)) throw new Error(`${url} did not answer with a page`);
805
+ items.push(...page.items);
806
+ if (!page.hasMore || page.items.length === 0) return items;
807
+ }
808
+ logger$5.warn(`${url} still had more pieces after ${REGISTRY_MAX_PAGES} pages`);
809
+ return items;
810
+ }
793
811
  function asList(value) {
794
812
  return Array.isArray(value) ? value : [];
795
813
  }
@@ -799,7 +817,7 @@ function asError(reason) {
799
817
  async function publishedLists(suggestions, timeoutMs) {
800
818
  const source = pieceRegistrySource();
801
819
  const cloudUrl = suggestions ? `${CATALOG_URL}?suggestionType=ACTION_AND_TRIGGER` : CATALOG_URL;
802
- const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchJson(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
820
+ const [fromRegistry, fromCloud] = await Promise.allSettled([source ? fetchRegistryCatalog(source.catalogUrl(suggestions), timeoutMs) : Promise.resolve([]), fetchJson(cloudUrl, timeoutMs)]);
803
821
  const registry = fromRegistry.status === "fulfilled" ? asList(fromRegistry.value).filter((entry) => typeof entry.name === "string") : [];
804
822
  if (fromRegistry.status === "rejected") logger$5.warn(`Piece registry ${source?.baseUrl ?? "?"} did not answer: ${String(fromRegistry.reason)}`);
805
823
  if (fromCloud.status === "rejected") {
@@ -849,19 +867,19 @@ async function fetchPieceCatalog() {
849
867
  };
850
868
  return value;
851
869
  }
852
- function registrySilentOn(error) {
853
- return error instanceof CatalogStatusError && error.status === 404;
854
- }
855
870
  async function fetchPieceJson(packageName, version) {
856
871
  const source = pieceRegistrySource();
857
872
  if (source) try {
858
873
  return await fetchJson(atVersion(source.pieceUrl(packageName), version));
859
874
  } catch (error) {
860
- if (!registrySilentOn(error)) throw error;
861
- logger$5.debug(`${source.baseUrl} does not serve "${packageName}": ${String(error)}`);
875
+ if (error instanceof CatalogStatusError && error.status === 404) logger$5.debug("@registry does not serve @piece", source.baseUrl, packageName);
876
+ else logger$5.warn("Piece registry @registry did not answer for @piece: @error", source.baseUrl, packageName, String(error));
862
877
  }
863
878
  return fetchJson(pieceUrl(packageName, version));
864
879
  }
880
+ function ttlFor(version) {
881
+ return version ? PINNED_TTL_MS : CACHE_TTL_MS;
882
+ }
865
883
  const detailCache = /* @__PURE__ */ new Map();
866
884
  async function fetchPieceDetail(packageName, version) {
867
885
  const key = `${packageName}@${version ?? ""}`;
@@ -870,7 +888,7 @@ async function fetchPieceDetail(packageName, version) {
870
888
  const value = await fetchPieceJson(packageName, version);
871
889
  detailCache.set(key, {
872
890
  value,
873
- expiresAt: Date.now() + CACHE_TTL_MS
891
+ expiresAt: Date.now() + ttlFor(version)
874
892
  });
875
893
  return value;
876
894
  }
@@ -899,7 +917,7 @@ async function fetchPieceTriggers(packageName, requested) {
899
917
  };
900
918
  triggersCache.set(key, {
901
919
  value,
902
- expiresAt: Date.now() + CACHE_TTL_MS
920
+ expiresAt: Date.now() + ttlFor(requested)
903
921
  });
904
922
  return value;
905
923
  }
@@ -928,7 +946,7 @@ async function fetchPieceActions(packageName, requested) {
928
946
  };
929
947
  actionsCache.set(key, {
930
948
  value,
931
- expiresAt: Date.now() + CACHE_TTL_MS
949
+ expiresAt: Date.now() + ttlFor(requested)
932
950
  });
933
951
  return value;
934
952
  }
@@ -1371,15 +1389,29 @@ function documentSummary(document, withState) {
1371
1389
  ...withState ? { state: globalState } : {}
1372
1390
  };
1373
1391
  }
1374
- function assertOperationsApplied(document, dispatched) {
1375
- const perScope = /* @__PURE__ */ new Map();
1376
- for (const action of dispatched) {
1377
- const scope = action.scope ?? "global";
1378
- perScope.set(scope, (perScope.get(scope) ?? 0) + 1);
1379
- }
1380
- const failed = [...perScope].flatMap(([scope, count]) => [...document.operations[scope] ?? []].sort((a, b) => a.index - b.index).slice(-count).filter((operation) => operation.error !== void 0));
1381
- if (failed.length === 0) return;
1382
- throw new Error(failed.map((operation) => `Action ${operation.action.type} failed: ${operation.error ?? "unknown error"}`).join("; "));
1392
+ const JOB_NOT_FOUND = "Job not found";
1393
+ function jobState(job) {
1394
+ if (job.status === JobStatus.FAILED && job.error?.message === JOB_NOT_FOUND) return {
1395
+ jobId: job.id,
1396
+ status: "UNKNOWN"
1397
+ };
1398
+ return {
1399
+ jobId: job.id,
1400
+ status: job.status,
1401
+ ...job.error ? { error: job.error.message } : {},
1402
+ ...job.result ? { actions: job.result.actions.map((action) => ({
1403
+ actionId: action.actionId,
1404
+ kind: action.kind,
1405
+ ...action.kind === "reducer-error" ? { message: action.message } : {},
1406
+ ...action.kind === "denied" ? { reason: action.reason } : {}
1407
+ })) } : {}
1408
+ };
1409
+ }
1410
+ function jobIds(jobs) {
1411
+ return Object.entries(jobs).sort(([a], [b]) => Number(b === "create") - Number(a === "create")).map(([, job]) => job.id);
1412
+ }
1413
+ function buildActions(input) {
1414
+ return input.actions.map((entry) => createAction$1(entry.type, entry.input, void 0, void 0, entry.scope ?? "global"));
1383
1415
  }
1384
1416
  var SubgraphReactorPort = class {
1385
1417
  host;
@@ -1410,7 +1442,7 @@ var SubgraphReactorPort = class {
1410
1442
  };
1411
1443
  }
1412
1444
  async get(input) {
1413
- const document = await this.client.get(input.documentId);
1445
+ const document = await this.client.get(input.documentId, input.branch ? { branch: input.branch } : void 0);
1414
1446
  return this.handOver(documentSummary(document, true));
1415
1447
  }
1416
1448
  async find(input) {
@@ -1432,27 +1464,61 @@ var SubgraphReactorPort = class {
1432
1464
  return true;
1433
1465
  }).filter((document) => matchesState(document, input.match)).map((document) => documentSummary(document, input.withState === true)));
1434
1466
  }
1435
- async create(input) {
1467
+ async submitCreate(input) {
1436
1468
  const target = input.parentId ? await this.resolveDriveTarget(input.parentId) : null;
1437
1469
  if (!target) {
1438
- const created = await this.client.createEmpty(input.documentType, { parentIdentifier: input.parentId });
1439
- if (!input.name) return this.handOver(documentSummary(created, true));
1440
- const naming = createAction$1("SET_NAME", { name: input.name });
1441
- const named = await this.client.execute(created.header.id, "main", [naming]);
1442
- assertOperationsApplied(named, [naming]);
1443
- return this.handOver(documentSummary(named, true));
1470
+ const { jobs } = await this.client.createEmptyAsync(input.documentType, { parentIdentifier: input.parentId });
1471
+ const documentId = jobs.create.documentId;
1472
+ return {
1473
+ documentId,
1474
+ jobIds: jobIds(jobs),
1475
+ followUps: input.name ? [{
1476
+ documentId,
1477
+ actions: [{
1478
+ type: "SET_NAME",
1479
+ input: { name: input.name }
1480
+ }]
1481
+ }] : []
1482
+ };
1444
1483
  }
1445
1484
  const module = await this.client.getDocumentModelModule(input.documentType);
1446
1485
  const empty = withSignaturePolicy(module.utils.createDocument(), await this.client.getCreateSignaturePolicy());
1447
1486
  if (input.name) empty.header.name = input.name;
1448
- const created = await this.client.drives.addFile(target.driveId, empty, target.parentFolder);
1449
- return this.handOver(documentSummary(created, true));
1487
+ const documentId = empty.header.id;
1488
+ const { jobs } = await this.client.createAsync(empty, target.driveId);
1489
+ const { type, input: node } = addFile({
1490
+ id: documentId,
1491
+ name: empty.header.name || documentId,
1492
+ documentType: input.documentType,
1493
+ parentFolder: target.parentFolder
1494
+ });
1495
+ return {
1496
+ documentId,
1497
+ jobIds: jobIds(jobs),
1498
+ followUps: [{
1499
+ documentId: target.driveId,
1500
+ actions: [{
1501
+ type,
1502
+ input: node
1503
+ }]
1504
+ }]
1505
+ };
1450
1506
  }
1451
- async execute(input) {
1452
- const actions = input.actions.map((entry) => createAction$1(entry.type, entry.input, void 0, void 0, entry.scope ?? "global"));
1453
- const document = await this.client.execute(input.documentId, input.branch ?? "main", actions);
1454
- assertOperationsApplied(document, input.actions);
1455
- return this.handOver(documentSummary(document, true));
1507
+ async submit(input) {
1508
+ const actions = buildActions(input);
1509
+ return {
1510
+ jobId: (await this.client.executeAsync(input.documentId, input.branch ?? "main", actions)).id,
1511
+ actionIds: actions.map((action) => action.id)
1512
+ };
1513
+ }
1514
+ async wait(input) {
1515
+ const signal = AbortSignal.timeout(input.maxWaitMs);
1516
+ try {
1517
+ return jobState(await this.client.waitForJob(input.jobId, signal));
1518
+ } catch (error) {
1519
+ if (!signal.aborted) throw error;
1520
+ }
1521
+ return jobState(await this.client.getJobStatus(input.jobId));
1456
1522
  }
1457
1523
  async handOver(summaries) {
1458
1524
  const record = currentDocumentRecorder();
@@ -1516,10 +1582,13 @@ var ScopedDesignTimeReactorPort = class {
1516
1582
  const allowed = await Promise.all(found.map((document) => this.canRead(document.documentId)));
1517
1583
  return found.filter((_, index) => allowed[index]);
1518
1584
  }
1519
- create(_input) {
1585
+ submit(_input) {
1520
1586
  return Promise.reject(/* @__PURE__ */ new Error(DESIGN_TIME_WRITES_REFUSED));
1521
1587
  }
1522
- execute(_input) {
1588
+ submitCreate(_input) {
1589
+ return Promise.reject(/* @__PURE__ */ new Error(DESIGN_TIME_WRITES_REFUSED));
1590
+ }
1591
+ wait(_input) {
1523
1592
  return Promise.reject(/* @__PURE__ */ new Error(DESIGN_TIME_WRITES_REFUSED));
1524
1593
  }
1525
1594
  async canRead(documentId) {
@@ -6864,8 +6933,47 @@ async function up(db) {
6864
6933
  try {
6865
6934
  await db.schema.dropTable("webhook_endpoint").ifExists().execute();
6866
6935
  } catch {}
6936
+ await migrateBlockType(db, "trigger_state", "trigger_name");
6937
+ await migrateBlockType(db, "step_execution", "block_name");
6867
6938
  return migrateTriggerStoreState(db);
6868
6939
  }
6940
+ const LEGACY_CORE_PIECE = "core";
6941
+ function legacyBlockIdentity(blockType) {
6942
+ const separator = blockType.lastIndexOf("#");
6943
+ if (separator <= 0) return {
6944
+ pieceName: blockType,
6945
+ name: ""
6946
+ };
6947
+ const spec = blockType.slice(0, separator);
6948
+ const fragment = blockType.slice(separator + 1);
6949
+ const name = fragment.startsWith("trigger:") ? fragment.slice(8) : fragment;
6950
+ const versionAt = spec.indexOf("@", 1);
6951
+ const pieceName = versionAt > 0 ? spec.slice(0, versionAt) : spec;
6952
+ return {
6953
+ pieceName: pieceName === LEGACY_CORE_PIECE ? CORE_PIECE_NAME : pieceName,
6954
+ name
6955
+ };
6956
+ }
6957
+ async function migrateBlockType(db, table, nameColumn) {
6958
+ const legacy = db;
6959
+ try {
6960
+ await legacy.schema.alterTable(table).alterColumn("block_type", (col) => col.dropNotNull()).execute();
6961
+ } catch {
6962
+ return;
6963
+ }
6964
+ for (const column of ["piece_name", nameColumn]) try {
6965
+ await legacy.schema.alterTable(table).addColumn(column, "text").execute();
6966
+ } catch {}
6967
+ const key = table === "trigger_state" ? "workflow_id" : "id";
6968
+ const rows = await legacy.selectFrom(table).select([key, "block_type"]).where((eb) => eb.or([eb("piece_name", "is", null), eb(nameColumn, "is", null)])).execute();
6969
+ for (const row of rows) {
6970
+ const { pieceName, name } = legacyBlockIdentity(row.block_type ?? "");
6971
+ await legacy.updateTable(table).set({
6972
+ piece_name: pieceName,
6973
+ [nameColumn]: name
6974
+ }).where(key, "=", row[key]).execute();
6975
+ }
6976
+ }
6869
6977
  async function migrateTriggerStoreState(db) {
6870
6978
  const unmigrated = /* @__PURE__ */ new Set();
6871
6979
  let rows;
@@ -7021,6 +7129,39 @@ function jsonOrNull(value) {
7021
7129
  return null;
7022
7130
  }
7023
7131
  }
7132
+ function triggerDocumentIds(payload) {
7133
+ if (payload === null || typeof payload !== "object") return [];
7134
+ const record = payload;
7135
+ return [...new Set([record.documentId, record.driveId].filter((id) => typeof id === "string" && id !== ""))];
7136
+ }
7137
+ function journaledTriggerDocumentIds(payload) {
7138
+ if (payload === null) return [];
7139
+ try {
7140
+ return triggerDocumentIds(JSON.parse(payload));
7141
+ } catch {
7142
+ return [];
7143
+ }
7144
+ }
7145
+ const TEST_TRIGGER_KIND = "test";
7146
+ function journaledPayloadNames(json, ids) {
7147
+ if (json === null) return false;
7148
+ let value;
7149
+ try {
7150
+ value = JSON.parse(json);
7151
+ } catch {
7152
+ return false;
7153
+ }
7154
+ return (Array.isArray(value) ? value : [value]).some((item) => {
7155
+ if (item === null || typeof item !== "object") return false;
7156
+ const record = item;
7157
+ return [
7158
+ record.documentId,
7159
+ record.driveId,
7160
+ record.parentId
7161
+ ].some((id) => typeof id === "string" && ids.includes(id));
7162
+ });
7163
+ }
7164
+ const erasedRuns = /* @__PURE__ */ new Set();
7024
7165
  async function claimDedupeIn(db, workflowId, dedupeKey, ttlMs, nowIso, runId) {
7025
7166
  const cutoff = new Date(Date.parse(nowIso) - ttlMs).toISOString();
7026
7167
  await db.deleteFrom("trigger_dedupe").where("workflow_id", "=", workflowId).where("created_at", "<", cutoff).execute();
@@ -7162,6 +7303,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7162
7303
  return id;
7163
7304
  }
7164
7305
  async recordStep(runId, ordinal, step) {
7306
+ if (erasedRuns.has(runId)) return;
7165
7307
  const values = stepValues(runId, ordinal, step);
7166
7308
  const { run_id: _run, step_id: _step, ...mutable } = values;
7167
7309
  await this.db.insertInto("step_execution").values({
@@ -7171,6 +7313,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7171
7313
  }
7172
7314
  async finishRun(runId, result, executionOrder) {
7173
7315
  this.runsInFlight.delete(runId);
7316
+ if (erasedRuns.delete(runId)) return;
7174
7317
  if (result.steps.length > 0) try {
7175
7318
  await this.sweepSteps(runId, result, executionOrder);
7176
7319
  } catch (error) {
@@ -7220,6 +7363,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7220
7363
  }
7221
7364
  async failRun(runId, error) {
7222
7365
  this.runsInFlight.delete(runId);
7366
+ if (erasedRuns.delete(runId)) return;
7223
7367
  await this.db.updateTable("run").set({
7224
7368
  status: "FAILED",
7225
7369
  error: redactMessage(error),
@@ -7255,7 +7399,7 @@ var WorkflowRunStore = class WorkflowRunStore {
7255
7399
  return byRun;
7256
7400
  }
7257
7401
  async recordRunDocuments(runId, documentIds) {
7258
- if (documentIds.length === 0) return;
7402
+ if (documentIds.length === 0 || erasedRuns.has(runId)) return;
7259
7403
  await this.db.insertInto("run_document").values([...new Set(documentIds)].map((documentId) => ({
7260
7404
  run_id: runId,
7261
7405
  document_id: documentId
@@ -7374,6 +7518,39 @@ var WorkflowRunStore = class WorkflowRunStore {
7374
7518
  if (deleted < batchSize) return pruned;
7375
7519
  }
7376
7520
  }
7521
+ async eraseRunsForDocuments(documentIds) {
7522
+ const ids = [...new Set(documentIds)];
7523
+ const erased = {
7524
+ runs: 0,
7525
+ steps: 0,
7526
+ documents: 0,
7527
+ dedupeKeysUnlinked: 0
7528
+ };
7529
+ if (ids.length === 0) return erased;
7530
+ return this.db.transaction().execute(async (trx) => {
7531
+ const runIds = new Set((await trx.selectFrom("run_document").select("run_id").where("document_id", "in", ids).execute()).map((row) => row.run_id));
7532
+ const own = await trx.selectFrom("run").select("id").where("workflow_id", "in", ids).execute();
7533
+ for (const row of own) runIds.add(row.id);
7534
+ const named = await trx.selectFrom("run").select(["id", "trigger_payload"]).where((eb) => eb.or(ids.map((id) => eb(sql`strpos(trigger_payload, ${id})`, ">", 0)))).execute();
7535
+ for (const row of named) if (journaledPayloadNames(row.trigger_payload, ids)) runIds.add(row.id);
7536
+ const sampled = await trx.selectFrom("step_execution").innerJoin("run", "run.id", "step_execution.run_id").select(["step_execution.run_id as runId", "step_execution.output"]).where("run.trigger_kind", "=", TEST_TRIGGER_KIND).where((eb) => eb.or(ids.map((id) => eb(sql`strpos(step_execution.output, ${id})`, ">", 0)))).execute();
7537
+ for (const row of sampled) if (journaledPayloadNames(row.output, ids)) runIds.add(row.runId);
7538
+ let frontier = [...runIds];
7539
+ while (frontier.length > 0) {
7540
+ frontier = (await trx.selectFrom("run").select("id").where("rerun_of", "in", frontier).execute()).map((row) => row.id).filter((id) => !runIds.has(id));
7541
+ for (const id of frontier) runIds.add(id);
7542
+ }
7543
+ if (runIds.size === 0) return erased;
7544
+ const all = [...runIds];
7545
+ const live = await trx.selectFrom("run").select("id").where("id", "in", all).where("status", "in", ["RUNNING", PENDING_RUN_STATUS]).execute();
7546
+ for (const row of live) erasedRuns.add(row.id);
7547
+ erased.steps = (await trx.deleteFrom("step_execution").where("run_id", "in", all).returning("id").execute()).length;
7548
+ erased.documents = (await trx.deleteFrom("run_document").where("run_id", "in", all).returning("run_id").execute()).length;
7549
+ erased.dedupeKeysUnlinked = (await trx.updateTable("trigger_dedupe").set({ run_id: null }).where("run_id", "in", all).returning("dedupe_key").execute()).length;
7550
+ erased.runs = (await trx.deleteFrom("run").where("id", "in", all).returning("id").execute()).length;
7551
+ return erased;
7552
+ });
7553
+ }
7377
7554
  async pruneDedupe(cutoffIso) {
7378
7555
  return (await this.db.deleteFrom("trigger_dedupe").where("created_at", "<", cutoffIso).returning("dedupe_key").execute()).length;
7379
7556
  }
@@ -8320,24 +8497,24 @@ function operationKey(op) {
8320
8497
  return op.context.ordinal > 0 ? `o:${op.context.ordinal}` : `${op.context.documentId}:${op.context.scope}:${op.context.branch}:${op.operation.index}`;
8321
8498
  }
8322
8499
  const OPERATION_DEDUPE_TTL_MS = 864e5;
8500
+ const STORE_REOPEN_MS = 1e3;
8501
+ const MAX_STORE_REOPEN_MS = 6e4;
8502
+ const UNJOURNALED_FIRES_LIMIT = 65536;
8323
8503
  function stringField(record, key) {
8324
8504
  const value = record[key];
8325
8505
  return typeof value === "string" && value !== "" ? value : void 0;
8326
8506
  }
8327
- function journaledTriggerDocumentIds(payload) {
8328
- if (payload === null) return [];
8329
- try {
8330
- return triggerDocumentIds(JSON.parse(payload));
8331
- } catch {
8332
- return [];
8333
- }
8334
- }
8335
- function triggerDocumentIds(payload) {
8336
- const record = inputRecord(payload);
8337
- return [...new Set([stringField(record, "documentId"), stringField(record, "driveId")].filter((id) => id !== void 0))];
8338
- }
8507
+ const ABSENT_ERROR_NAMES = /* @__PURE__ */ new Set([
8508
+ "DocumentNotFoundError",
8509
+ "DocumentPurgedError",
8510
+ "DocumentDeletedError"
8511
+ ]);
8339
8512
  function isAbsent(error) {
8340
- return error instanceof Error && (error.name === "DocumentNotFoundError" || error.name === "DocumentDeletedError");
8513
+ return error instanceof Error && ABSENT_ERROR_NAMES.has(error.name);
8514
+ }
8515
+ function purgedWorkflowIds(markers) {
8516
+ const ids = markers.filter(({ operation, context }) => (stringField(inputRecord(operation.action.input), "documentType") ?? context.documentType) === WORKFLOW_DOCUMENT_TYPE).map(({ context }) => context.documentId);
8517
+ return [...new Set(ids)];
8341
8518
  }
8342
8519
  function inputRecord(input) {
8343
8520
  if (input === null || typeof input !== "object") return {};
@@ -8451,7 +8628,6 @@ function registrationKey(state) {
8451
8628
  trigger: trigger && armed
8452
8629
  });
8453
8630
  }
8454
- const TEST_TRIGGER_KIND = "test";
8455
8631
  function configRecord(config) {
8456
8632
  if (config && typeof config === "object" && !Array.isArray(config)) return config;
8457
8633
  return {};
@@ -8466,6 +8642,11 @@ var WorkflowRuntimeService = class {
8466
8642
  executor;
8467
8643
  pieceWorkers;
8468
8644
  storePromise;
8645
+ storeError;
8646
+ storeOpening = false;
8647
+ storeReopenAt = 0;
8648
+ storeReopenMs = STORE_REOPEN_MS;
8649
+ retentionSweep;
8469
8650
  secretsPromise;
8470
8651
  oauthAttemptsPromise;
8471
8652
  tokenRefresher;
@@ -8481,10 +8662,7 @@ var WorkflowRuntimeService = class {
8481
8662
  });
8482
8663
  this.logger = host.logger ?? logger;
8483
8664
  this.attachments = host.attachments ? createAttachmentPort(host.attachments, () => currentWorkflowId(), (documentId, ref) => host.canReadAttachmentRef?.(documentId, ref) ?? Promise.resolve(false)) : void 0;
8484
- this.storePromise = WorkflowRunStore.create(host.relationalDb);
8485
- this.storePromise.catch((error) => {
8486
- this.logger.error("Failed to open the workflow run store: @error", error);
8487
- });
8665
+ this.storePromise = this.openStore();
8488
8666
  this.seedPromise = this.seedWithRetries();
8489
8667
  this.startRetention();
8490
8668
  }
@@ -8497,7 +8675,7 @@ var WorkflowRuntimeService = class {
8497
8675
  };
8498
8676
  this.retentionTimer = setInterval(sweep, RETENTION_SWEEP_INTERVAL_MS);
8499
8677
  this.retentionTimer.unref();
8500
- this.storePromise.then(sweep, () => void 0);
8678
+ this.retentionSweep = sweep;
8501
8679
  }
8502
8680
  async sweepRetention(retentionMs) {
8503
8681
  const store = await this.store();
@@ -8513,12 +8691,31 @@ var WorkflowRuntimeService = class {
8513
8691
  }
8514
8692
  }
8515
8693
  async store() {
8694
+ if (this.storeError !== void 0 && !this.storeOpening && Date.now() >= this.storeReopenAt) this.storePromise = this.openStore();
8516
8695
  try {
8517
8696
  return await this.storePromise;
8518
8697
  } catch {
8519
8698
  return;
8520
8699
  }
8521
8700
  }
8701
+ openStore() {
8702
+ this.storeOpening = true;
8703
+ const opening = WorkflowRunStore.create(this.host.relationalDb);
8704
+ opening.then(() => {
8705
+ this.storeOpening = false;
8706
+ if (this.storeError !== void 0) this.logger.info("Workflow run store opened after failing to");
8707
+ this.storeError = void 0;
8708
+ this.storeReopenMs = STORE_REOPEN_MS;
8709
+ this.retentionSweep?.();
8710
+ }, (error) => {
8711
+ this.storeOpening = false;
8712
+ this.storeError = error;
8713
+ this.storeReopenAt = Date.now() + this.storeReopenMs;
8714
+ this.logger.error("Failed to open the workflow run store, retrying in @ms ms: @error", this.storeReopenMs, error);
8715
+ this.storeReopenMs = Math.min(this.storeReopenMs * 2, MAX_STORE_REOPEN_MS);
8716
+ });
8717
+ return opening;
8718
+ }
8522
8719
  secrets() {
8523
8720
  this.secretsPromise ??= this.host.secrets !== void 0 ? Promise.resolve(this.host.secrets) : LocalEncryptedSecretStore.create(this.host.relationalDb, { keyFile: this.host.secretsKeyFile });
8524
8721
  return this.secretsPromise;
@@ -8846,6 +9043,18 @@ var WorkflowRuntimeService = class {
8846
9043
  }
8847
9044
  return false;
8848
9045
  }
9046
+ async onDocumentsPurged(markers) {
9047
+ const documentIds = [...new Set(markers.map((marker) => marker.context.documentId))];
9048
+ const store = await this.store();
9049
+ if (!store) {
9050
+ this.logger.error("Run journal unavailable; purged documents @ids are not erased and the triggers cursor holds until it reopens: @error", documentIds, this.storeError);
9051
+ throw new Error("Erasing purged documents needs the run journal", { cause: this.storeError });
9052
+ }
9053
+ for (const workflowId of purgedWorkflowIds(markers)) await this.erasePurgedWorkflow(store, workflowId);
9054
+ const erased = await store.eraseRunsForDocuments(documentIds);
9055
+ if (erased.runs > 0) this.logger.info("Erased @runs workflow run(s) of purged documents @ids", erased.runs, documentIds);
9056
+ return erased;
9057
+ }
8849
9058
  async onOperations(operations) {
8850
9059
  const hints = collectLifecycleParentHints(operations);
8851
9060
  for (const { operation, context } of operations) {
@@ -8887,10 +9096,31 @@ var WorkflowRuntimeService = class {
8887
9096
  }, opKey);
8888
9097
  }
8889
9098
  }
9099
+ unjournaledFires = /* @__PURE__ */ new Set();
9100
+ fireUnjournaled(fireKey, workflowId, payload, kind) {
9101
+ this.unjournaledFires.add(fireKey);
9102
+ if (this.unjournaledFires.size > UNJOURNALED_FIRES_LIMIT) {
9103
+ const oldest = this.unjournaledFires.values().next().value;
9104
+ this.unjournaledFires.delete(oldest);
9105
+ this.logger.warn("Too many unjournaled workflow fires held; a redelivery of the oldest fires again");
9106
+ }
9107
+ this.fireFromTrigger(workflowId, payload, kind);
9108
+ }
8890
9109
  async enqueueFire(workflowId, payload, kind, opKey) {
9110
+ const fireKey = JSON.stringify([workflowId, opKey]);
8891
9111
  const store = await this.store();
9112
+ if (this.unjournaledFires.has(fireKey)) {
9113
+ if (!store) return;
9114
+ try {
9115
+ await store.claimDedupe(workflowId, `op:${opKey}`, OPERATION_DEDUPE_TTL_MS, (/* @__PURE__ */ new Date()).toISOString());
9116
+ this.unjournaledFires.delete(fireKey);
9117
+ } catch (error) {
9118
+ this.logger.warn(`Could not record the unjournaled ${kind} fire for workflow ${workflowId}: @error`, error);
9119
+ }
9120
+ return;
9121
+ }
8892
9122
  if (!store) {
8893
- this.fireFromTrigger(workflowId, payload, kind);
9123
+ this.fireUnjournaled(fireKey, workflowId, payload, kind);
8894
9124
  return;
8895
9125
  }
8896
9126
  let runId;
@@ -8902,7 +9132,7 @@ var WorkflowRuntimeService = class {
8902
9132
  });
8903
9133
  } catch (error) {
8904
9134
  this.logger.error(`Could not journal the ${kind} fire for workflow ${workflowId}; running it without a durable record`, error);
8905
- this.fireFromTrigger(workflowId, payload, kind);
9135
+ this.fireUnjournaled(fireKey, workflowId, payload, kind);
8906
9136
  return;
8907
9137
  }
8908
9138
  if (runId === null) return;
@@ -8978,14 +9208,27 @@ var WorkflowRuntimeService = class {
8978
9208
  }
8979
9209
  }
8980
9210
  disarmDeleted(workflowId) {
9211
+ this.dropDeleted(workflowId);
9212
+ this.releaseDeleted(workflowId).catch((error) => {
9213
+ this.logger.error(`Could not disarm deleted workflow ${workflowId}`, error);
9214
+ });
9215
+ }
9216
+ dropDeleted(workflowId) {
8981
9217
  this.registry.delete(workflowId);
8982
9218
  this.registeredAs.delete(workflowId);
8983
9219
  this.unarmed.delete(workflowId);
8984
9220
  this.cancelResolutionRetry(workflowId);
8985
9221
  this.cancelTriggerTest(workflowId, "stopped: the workflow was deleted");
8986
- this.supervisor().forget(workflowId).then(async () => (await this.endpoints())?.revoke(workflowId)).catch((error) => {
8987
- this.logger.error(`Could not disarm deleted workflow ${workflowId}`, error);
8988
- });
9222
+ }
9223
+ async releaseDeleted(workflowId) {
9224
+ await this.supervisor().forget(workflowId);
9225
+ await (await this.endpoints())?.revoke(workflowId);
9226
+ }
9227
+ async erasePurgedWorkflow(store, workflowId) {
9228
+ this.dropDeleted(workflowId);
9229
+ await this.releaseDeleted(workflowId);
9230
+ await store.deleteDedupe(workflowId);
9231
+ for (const scope of ["FLOW", "PROJECT"]) await store.deletePieceStore(scope, testPartitionKey(scope, workflowId));
8989
9232
  }
8990
9233
  async matchDocumentLifecycle(operation, context, hints, opKey) {
8991
9234
  const kind = lifecycleKindForDocumentAction(operation.action.type);
@@ -9118,13 +9361,17 @@ var WorkflowRuntimeService = class {
9118
9361
  * undefined means the workflow is not armed, answered exactly as an unknown token is. */
9119
9362
  async webhookPolicy(workflowId) {
9120
9363
  await this.seedPromise;
9364
+ const registration = this.liveWebhook(workflowId);
9365
+ if (registration) {
9366
+ if (registration.kind === "piece-webhook") return {};
9367
+ return this.policyOf(registration.config, workflowId);
9368
+ }
9121
9369
  const test = this.webhookTests.get(workflowId);
9122
- if (test) return this.policyOf(test.config, workflowId);
9370
+ return test ? this.policyOf(test.config, workflowId) : void 0;
9371
+ }
9372
+ liveWebhook(workflowId) {
9123
9373
  const registration = this.registry.get(workflowId);
9124
- if (!registration) return void 0;
9125
- if (registration.kind === "piece-webhook") return {};
9126
- if (registration.kind !== "webhook") return void 0;
9127
- return this.policyOf(registration.config, workflowId);
9374
+ return registration?.kind === "piece-webhook" || registration?.kind === "webhook" ? registration : void 0;
9128
9375
  }
9129
9376
  async policyOf(config, workflowId) {
9130
9377
  return {
@@ -9168,17 +9415,16 @@ var WorkflowRuntimeService = class {
9168
9415
  * answered any challenge for; all that is left is deciding what it means. */
9169
9416
  async deliverWebhook(request) {
9170
9417
  const workflowId = request.key;
9418
+ const registration = this.liveWebhook(workflowId);
9171
9419
  const test = this.webhookTests.get(workflowId);
9172
9420
  if (test) {
9173
9421
  this.webhookTests.delete(workflowId);
9174
9422
  clearTimeout(test.timer);
9175
9423
  test.resolve(webhookPayload(request));
9176
- return { status: test.config.responseStatus };
9424
+ if (!registration) return { status: test.config.responseStatus };
9177
9425
  }
9178
- const registration = this.registry.get(workflowId);
9179
9426
  if (!registration) return UNAUTHORIZED;
9180
9427
  if (registration.kind === "piece-webhook") return this.deliverToPiece(registration.binding, request);
9181
- if (registration.kind !== "webhook") return UNAUTHORIZED;
9182
9428
  const { config } = registration;
9183
9429
  const payload = webhookPayload(request);
9184
9430
  if (config.responseMode === "async") {
@@ -10145,6 +10391,7 @@ var WorkflowRuntimeService = class {
10145
10391
  if (!run) throw new Error(`Run "${runId}" not found`);
10146
10392
  await this.assertCanReadDocument(run.workflow_id, ctx);
10147
10393
  if (!ctx || (await this.servedRuns([run], ctx)).length === 0) throw new Error(`Run "${runId}" not found`);
10394
+ if (run.trigger_kind === "test") throw new Error(`Run "${runId}" tested a single step or trigger; test it again instead of rerunning it`);
10148
10395
  if (run.status !== "FAILED") throw new Error(`Only FAILED runs can be rerun; run is ${run.status}`);
10149
10396
  const triggerPayload = run.trigger_payload === null ? void 0 : JSON.parse(run.trigger_payload);
10150
10397
  if (containsRedactedMarker(triggerPayload)) throw new Error(`Trigger payload of run "${runId}" was redacted and cannot be replayed; fire the workflow again instead of rerunning it`);
@@ -10196,7 +10443,7 @@ var WorkflowRuntimeService = class {
10196
10443
  };
10197
10444
  let input;
10198
10445
  try {
10199
- input = resolveExpressions(step.config, {
10446
+ input = resolveStepInput(definition.steps[0], {
10200
10447
  trigger: { payload: upstream.triggerPayload },
10201
10448
  steps: upstream.priorSteps,
10202
10449
  variables: Object.fromEntries(variables.map((v) => [v.key, v.value]))
@@ -10426,12 +10673,19 @@ var WorkflowTriggersReadModel = class extends BaseReadModel {
10426
10673
  readModelId: WORKFLOW_TRIGGERS_READ_MODEL,
10427
10674
  rebuildStateOnInit: false,
10428
10675
  indexing: defaultReadModelIndexingConfig,
10429
- startFrom: "head"
10676
+ startFrom: "head",
10677
+ purgeFence: "skip"
10430
10678
  });
10431
10679
  this.runtime = runtime;
10432
10680
  }
10433
10681
  async commitOperations(items) {
10434
- await this.runtime.onOperations(items);
10682
+ const markers = items.filter((item) => isPurgeMarker(item.operation));
10683
+ if (markers.length === 0) {
10684
+ await this.runtime.onOperations(items);
10685
+ return;
10686
+ }
10687
+ await this.runtime.onOperations(items.filter((item) => !isPurgeMarker(item.operation)));
10688
+ await this.runtime.onDocumentsPurged(markers);
10435
10689
  }
10436
10690
  };
10437
10691
  //#endregion