octaflow 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -0
- package/dist/ai/index.d.ts +1 -1
- package/dist/ai/providers/index.js +1 -0
- package/dist/drizzle/index.d.ts +173 -0
- package/dist/drizzle/index.js +199 -0
- package/package.json +13 -2
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -230,6 +230,9 @@ what the layers you import need:
|
|
|
230
230
|
# Postgres store / gate / event sink
|
|
231
231
|
pnpm add pg
|
|
232
232
|
|
|
233
|
+
# the same tables as Drizzle column sets, for a host that lets drizzle-kit own its migrations
|
|
234
|
+
pnpm add drizzle-orm
|
|
235
|
+
|
|
233
236
|
# pg-boss dispatcher, workers, cron scheduler
|
|
234
237
|
pnpm add pg-boss
|
|
235
238
|
|
|
@@ -240,6 +243,63 @@ pnpm add ai @ai-sdk/provider
|
|
|
240
243
|
> Pure in-memory usage (great for tests and single-process apps) needs **nothing** beyond
|
|
241
244
|
> `zod` — the engine, `defineStep`/`buildWorkflow`, and the in-memory store are all in the core.
|
|
242
245
|
|
|
246
|
+
### The Postgres tables
|
|
247
|
+
|
|
248
|
+
`octaflow/postgres` owns five tables and ships their DDL: `flowStoreDdl()` (`flow_workflow`,
|
|
249
|
+
`flow_workflow_step`), `flowEventDdl()` (`flow_step_event`) and `flowGateDdl()`
|
|
250
|
+
(`flow_rate_bucket`, `flow_step_lease`). Each is idempotent `CREATE TABLE IF NOT EXISTS`, takes
|
|
251
|
+
a `schema`, and is meant to be pasted into your own migration system; `applySchema(pool, ddl)`
|
|
252
|
+
is the dev and test shortcut.
|
|
253
|
+
|
|
254
|
+
#### The Drizzle columns
|
|
255
|
+
|
|
256
|
+
The rule for every table-owning capability on the platform: `./postgres` ships the store over
|
|
257
|
+
the structural executor plus its `xDdl()`, and `./drizzle` ships the same table as a spreadable
|
|
258
|
+
column set, so a host on Drizzle declares it in its own schema and drizzle-kit owns the
|
|
259
|
+
migration. Where the store's queries are one SQL statement by nature — every one of octaflow's
|
|
260
|
+
is — `./drizzle` ships the columns only, and the store stays the SQL one. So
|
|
261
|
+
`octaflow/drizzle` exports `flowWorkflowColumns`, `flowWorkflowStepColumns`,
|
|
262
|
+
`flowStepEventColumns`, `flowRateBucketColumns` and `flowStepLeaseColumns` (`drizzle-orm` an
|
|
263
|
+
optional peer) and nothing that runs a query. The foreign keys, the composite keys, the unique
|
|
264
|
+
constraint and the indexes are yours to declare on the `pgTable`; each column set's doc lists
|
|
265
|
+
the ones its DDL creates, and two of them the store relies on:
|
|
266
|
+
|
|
267
|
+
```ts
|
|
268
|
+
import { sql } from 'drizzle-orm';
|
|
269
|
+
import { foreignKey, index, pgTable, primaryKey, unique, uniqueIndex } from 'drizzle-orm/pg-core';
|
|
270
|
+
import { flowWorkflowColumns, flowWorkflowStepColumns, flowStepLeaseColumns } from 'octaflow/drizzle';
|
|
271
|
+
|
|
272
|
+
export const flowWorkflow = pgTable('flow_workflow', flowWorkflowColumns, (t) => [
|
|
273
|
+
index('flow_workflow_partition_status_idx').on(t.partitionKey, t.status),
|
|
274
|
+
index('flow_workflow_partition_type_idx').on(t.partitionKey, t.type),
|
|
275
|
+
index('flow_workflow_partition_entity_idx').on(t.partitionKey, t.entityRef),
|
|
276
|
+
index('flow_workflow_parent_idx').on(t.parentWorkflowId),
|
|
277
|
+
// The deadline sweep reads only live runs that have one.
|
|
278
|
+
index('flow_workflow_deadline_idx').on(t.deadlineAt).where(sql`deadline_at IS NOT NULL`),
|
|
279
|
+
// `start` with an idempotency key upserts against this index: without it a repeated start is a second run.
|
|
280
|
+
uniqueIndex('flow_workflow_idempotency_idx').on(t.partitionKey, t.idempotencyKey).where(sql`idempotency_key IS NOT NULL`),
|
|
281
|
+
]);
|
|
282
|
+
|
|
283
|
+
export const flowWorkflowStep = pgTable('flow_workflow_step', flowWorkflowStepColumns, (t) => [
|
|
284
|
+
foreignKey({ columns: [t.workflowId], foreignColumns: [flowWorkflow.id] }).onDelete('cascade'),
|
|
285
|
+
foreignKey({ columns: [t.parentStepId], foreignColumns: [t.id] }).onDelete('cascade'),
|
|
286
|
+
unique('flow_workflow_step_workflow_id_key_unique').on(t.workflowId, t.key),
|
|
287
|
+
index('flow_workflow_step_workflow_idx').on(t.workflowId),
|
|
288
|
+
index('flow_workflow_step_status_idx').on(t.workflowId, t.status),
|
|
289
|
+
index('flow_workflow_step_parent_idx').on(t.parentStepId),
|
|
290
|
+
]);
|
|
291
|
+
|
|
292
|
+
export const flowStepLease = pgTable('flow_step_lease', flowStepLeaseColumns, (t) => [
|
|
293
|
+
primaryKey({ columns: [t.partitionKey, t.stepType, t.stepId] }),
|
|
294
|
+
index('flow_step_lease_active_idx').on(t.partitionKey, t.stepType, t.expiresAt),
|
|
295
|
+
]);
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
The store stays plain SQL, bound to the `schema` you hand `createPgWorkflowStore`,
|
|
299
|
+
`createPgStepGate` and `createPgEventSink` — and, for the gate, to the `tables` names — so
|
|
300
|
+
declare the tables under the same schema and names. A test in the package holds every column
|
|
301
|
+
set and its DDL equal, column for column: a column added to one side alone fails the build.
|
|
302
|
+
|
|
243
303
|
---
|
|
244
304
|
|
|
245
305
|
---
|
package/dist/ai/index.d.ts
CHANGED
|
@@ -58,7 +58,7 @@ export declare function createInstrumentedEmbeddingModel(model: EmbeddingModelV4
|
|
|
58
58
|
* (1 USD = 1,000,000 microdollars) to avoid float accumulation.
|
|
59
59
|
*
|
|
60
60
|
* The pricing table is injectable: pass your own to `createCostEstimator` for a
|
|
61
|
-
* different provider/model set. A default
|
|
61
|
+
* different provider/model set. A default table for the registry's vendors ships.
|
|
62
62
|
*/
|
|
63
63
|
export interface ModelPricing {
|
|
64
64
|
inputPerMillion: number;
|
|
@@ -125,6 +125,7 @@ function checkEndpoint(config) {
|
|
|
125
125
|
url = void 0;
|
|
126
126
|
}
|
|
127
127
|
if (url === void 0 || url.protocol !== "http:" && url.protocol !== "https:") throw new AiProviderConfigError("ai_base_url_invalid", `baseUrl must be an absolute http(s) URL, got ${JSON.stringify(config.baseUrl)}`);
|
|
128
|
+
if (url.protocol === "http:" && residency.value.region !== "self-hosted") throw new AiProviderConfigError("ai_base_url_invalid", `baseUrl must be https for a vendor endpoint (residency ${residency.value.region}); http is for self-hosted endpoints only, got ${JSON.stringify(config.baseUrl)}`);
|
|
128
129
|
return residency.value;
|
|
129
130
|
}
|
|
130
131
|
function requireModel(modelId, field) {
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
//#region src/drizzle/index.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* `flow_workflow` — one row per run. `partition_key` is the engine's scoping
|
|
4
|
+
* column: the store is bound to one partition and every query filters by it.
|
|
5
|
+
*
|
|
6
|
+
* The DDL adds, and the host declares:
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { index, pgTable, uniqueIndex } from 'drizzle-orm/pg-core';
|
|
10
|
+
* import { sql } from 'drizzle-orm';
|
|
11
|
+
* import { flowWorkflowColumns } from 'octaflow/drizzle';
|
|
12
|
+
*
|
|
13
|
+
* export const flowWorkflow = pgTable('flow_workflow', flowWorkflowColumns, (t) => [
|
|
14
|
+
* index('flow_workflow_partition_status_idx').on(t.partitionKey, t.status),
|
|
15
|
+
* index('flow_workflow_parent_idx').on(t.parentWorkflowId),
|
|
16
|
+
* index('flow_workflow_partition_type_idx').on(t.partitionKey, t.type),
|
|
17
|
+
* index('flow_workflow_partition_entity_idx').on(t.partitionKey, t.entityRef),
|
|
18
|
+
* // Load-bearing: the deadline sweep reads only live runs that have one.
|
|
19
|
+
* index('flow_workflow_deadline_idx').on(t.deadlineAt).where(sql`deadline_at IS NOT NULL`),
|
|
20
|
+
* // Load-bearing: `start` with an idempotency key relies on this unique index
|
|
21
|
+
* // to make a repeated start return the existing run instead of a second one.
|
|
22
|
+
* uniqueIndex('flow_workflow_idempotency_idx').on(t.partitionKey, t.idempotencyKey).where(sql`idempotency_key IS NOT NULL`),
|
|
23
|
+
* ]);
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* `parent_workflow_id` and `parent_step_id` carry no foreign key in the DDL
|
|
27
|
+
* (a parent may outlive or predate its child in a migration); add one if you
|
|
28
|
+
* want the constraint.
|
|
29
|
+
*/
|
|
30
|
+
export declare const flowWorkflowColumns: {
|
|
31
|
+
id: import("drizzle-orm").IsPrimaryKey<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigSerial53BuilderInitial<"id">>>;
|
|
32
|
+
partitionKey: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"partition_key", [string, ...string[]]>>;
|
|
33
|
+
/** The workflow definition's name. */
|
|
34
|
+
type: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"type", [string, ...string[]]>>;
|
|
35
|
+
status: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"status", [string, ...string[]]>>;
|
|
36
|
+
input: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"input">>>;
|
|
37
|
+
output: import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"output">;
|
|
38
|
+
error: import("drizzle-orm/pg-core").PgTextBuilderInitial<"error", [string, ...string[]]>;
|
|
39
|
+
/** The host's handle for what the run is about (`'record:88'`), for lookups by entity. */
|
|
40
|
+
entityRef: import("drizzle-orm/pg-core").PgTextBuilderInitial<"entity_ref", [string, ...string[]]>;
|
|
41
|
+
totalSteps: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"total_steps">>;
|
|
42
|
+
completedSteps: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"completed_steps">>>;
|
|
43
|
+
failedSteps: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"failed_steps">>>;
|
|
44
|
+
metadata: import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"metadata">;
|
|
45
|
+
/** Set on a `start` that asked for one; unique per partition where present. */
|
|
46
|
+
idempotencyKey: import("drizzle-orm/pg-core").PgTextBuilderInitial<"idempotency_key", [string, ...string[]]>;
|
|
47
|
+
/** The run's deadline, when the definition set one; the sweep reads it. */
|
|
48
|
+
deadlineAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
49
|
+
/** For a sub-workflow: the run that started it, and the step in that run. */
|
|
50
|
+
parentWorkflowId: import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"parent_workflow_id">;
|
|
51
|
+
parentStepId: import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"parent_step_id">;
|
|
52
|
+
createdAt: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>>>;
|
|
53
|
+
startedAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
54
|
+
completedAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* `flow_workflow_step` — one row per step of a run, including the children a
|
|
58
|
+
* fan-out creates at runtime.
|
|
59
|
+
*
|
|
60
|
+
* The DDL adds, and the host declares:
|
|
61
|
+
*
|
|
62
|
+
* ```ts
|
|
63
|
+
* import { foreignKey, index, pgTable, unique, type AnyPgColumn } from 'drizzle-orm/pg-core';
|
|
64
|
+
* import { flowWorkflowStepColumns } from 'octaflow/drizzle';
|
|
65
|
+
*
|
|
66
|
+
* export const flowWorkflowStep = pgTable('flow_workflow_step', flowWorkflowStepColumns, (t) => [
|
|
67
|
+
* foreignKey({ columns: [t.workflowId], foreignColumns: [flowWorkflow.id] }).onDelete('cascade'),
|
|
68
|
+
* foreignKey({ columns: [t.parentStepId], foreignColumns: [t.id] }).onDelete('cascade'),
|
|
69
|
+
* // `(workflow_id, key)` is the step's identity within a run; the constraint
|
|
70
|
+
* // is what refuses a second step under the same key.
|
|
71
|
+
* unique('flow_workflow_step_workflow_id_key_unique').on(t.workflowId, t.key),
|
|
72
|
+
* index('flow_workflow_step_workflow_idx').on(t.workflowId),
|
|
73
|
+
* index('flow_workflow_step_status_idx').on(t.workflowId, t.status),
|
|
74
|
+
* index('flow_workflow_step_parent_idx').on(t.parentStepId),
|
|
75
|
+
* ]);
|
|
76
|
+
* ```
|
|
77
|
+
*/
|
|
78
|
+
export declare const flowWorkflowStepColumns: {
|
|
79
|
+
id: import("drizzle-orm").IsPrimaryKey<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigSerial53BuilderInitial<"id">>>;
|
|
80
|
+
workflowId: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"workflow_id">>;
|
|
81
|
+
partitionKey: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"partition_key", [string, ...string[]]>>;
|
|
82
|
+
/** The step's key within its workflow — unique per run. */
|
|
83
|
+
key: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"key", [string, ...string[]]>>;
|
|
84
|
+
/** The step handler's name. */
|
|
85
|
+
type: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"type", [string, ...string[]]>>;
|
|
86
|
+
status: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"status", [string, ...string[]]>>;
|
|
87
|
+
/** The keys of the steps this one waits for. */
|
|
88
|
+
dependencies: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"dependencies">>>;
|
|
89
|
+
input: import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"input">;
|
|
90
|
+
output: import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"output">;
|
|
91
|
+
error: import("drizzle-orm/pg-core").PgTextBuilderInitial<"error", [string, ...string[]]>;
|
|
92
|
+
metadata: import("drizzle-orm/pg-core").PgJsonbBuilderInitial<"metadata">;
|
|
93
|
+
attempts: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"attempts">>>;
|
|
94
|
+
/** For a fan-out child: the step that spawned it. */
|
|
95
|
+
parentStepId: import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"parent_step_id">;
|
|
96
|
+
/** The last heartbeat a long step sent; the sweep uses it to tell stalled from slow. */
|
|
97
|
+
heartbeatAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
98
|
+
startedAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
99
|
+
completedAt: import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* `flow_step_event` — the append-only run history `createPgEventSink` writes,
|
|
103
|
+
* one row per engine event.
|
|
104
|
+
*
|
|
105
|
+
* The DDL adds, and the host declares:
|
|
106
|
+
*
|
|
107
|
+
* ```ts
|
|
108
|
+
* export const flowStepEvent = pgTable('flow_step_event', flowStepEventColumns, (t) => [
|
|
109
|
+
* index('flow_step_event_workflow_idx').on(t.partitionKey, t.workflowId, t.id),
|
|
110
|
+
* ]);
|
|
111
|
+
* ```
|
|
112
|
+
*
|
|
113
|
+
* `workflow_id` carries no foreign key on purpose: the history outlives the run.
|
|
114
|
+
*/
|
|
115
|
+
export declare const flowStepEventColumns: {
|
|
116
|
+
id: import("drizzle-orm").IsPrimaryKey<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigSerial53BuilderInitial<"id">>>;
|
|
117
|
+
partitionKey: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"partition_key", [string, ...string[]]>>;
|
|
118
|
+
workflowId: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"workflow_id">>;
|
|
119
|
+
workflowType: import("drizzle-orm/pg-core").PgTextBuilderInitial<"workflow_type", [string, ...string[]]>;
|
|
120
|
+
/** Null for a workflow-level event. */
|
|
121
|
+
stepId: import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"step_id">;
|
|
122
|
+
stepKey: import("drizzle-orm/pg-core").PgTextBuilderInitial<"step_key", [string, ...string[]]>;
|
|
123
|
+
stepType: import("drizzle-orm/pg-core").PgTextBuilderInitial<"step_type", [string, ...string[]]>;
|
|
124
|
+
eventType: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"event_type", [string, ...string[]]>>;
|
|
125
|
+
attempt: import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"attempt">;
|
|
126
|
+
durationMs: import("drizzle-orm/pg-core").PgIntegerBuilderInitial<"duration_ms">;
|
|
127
|
+
error: import("drizzle-orm/pg-core").PgTextBuilderInitial<"error", [string, ...string[]]>;
|
|
128
|
+
at: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>>;
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* `flow_rate_bucket` — one token bucket per `(partition_key, step_type)` for
|
|
132
|
+
* `createPgStepGate`'s rate limits. The table name is the DDL's default and
|
|
133
|
+
* the gate's `tables.rateBucket` option; the two must agree.
|
|
134
|
+
*
|
|
135
|
+
* The DDL adds a composite primary key, and the host declares it:
|
|
136
|
+
*
|
|
137
|
+
* ```ts
|
|
138
|
+
* export const flowRateBucket = pgTable('flow_rate_bucket', flowRateBucketColumns, (t) => [
|
|
139
|
+
* primaryKey({ columns: [t.partitionKey, t.stepType] }),
|
|
140
|
+
* ]);
|
|
141
|
+
* ```
|
|
142
|
+
*/
|
|
143
|
+
export declare const flowRateBucketColumns: {
|
|
144
|
+
partitionKey: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"partition_key", [string, ...string[]]>>;
|
|
145
|
+
stepType: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"step_type", [string, ...string[]]>>;
|
|
146
|
+
/** The bucket's fill, refilled from `updated_at` on every read. */
|
|
147
|
+
tokens: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgDoublePrecisionBuilderInitial<"tokens">>;
|
|
148
|
+
updatedAt: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>>>;
|
|
149
|
+
};
|
|
150
|
+
/**
|
|
151
|
+
* `flow_step_lease` — one row per running step under a concurrency cap, for
|
|
152
|
+
* `createPgStepGate`'s `maxConcurrent`. The table name is the DDL's default
|
|
153
|
+
* and the gate's `tables.lease` option; the two must agree.
|
|
154
|
+
*
|
|
155
|
+
* The DDL adds a composite primary key and one index, and the host declares
|
|
156
|
+
* them — the index name is derived from the table name in the DDL:
|
|
157
|
+
*
|
|
158
|
+
* ```ts
|
|
159
|
+
* export const flowStepLease = pgTable('flow_step_lease', flowStepLeaseColumns, (t) => [
|
|
160
|
+
* primaryKey({ columns: [t.partitionKey, t.stepType, t.stepId] }),
|
|
161
|
+
* index('flow_step_lease_active_idx').on(t.partitionKey, t.stepType, t.expiresAt),
|
|
162
|
+
* ]);
|
|
163
|
+
* ```
|
|
164
|
+
*/
|
|
165
|
+
export declare const flowStepLeaseColumns: {
|
|
166
|
+
partitionKey: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"partition_key", [string, ...string[]]>>;
|
|
167
|
+
stepType: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTextBuilderInitial<"step_type", [string, ...string[]]>>;
|
|
168
|
+
stepId: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgBigInt53BuilderInitial<"step_id">>;
|
|
169
|
+
acquiredAt: import("drizzle-orm").HasDefault<import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>>>;
|
|
170
|
+
/** A lease past this instant is reaped on the next acquire. */
|
|
171
|
+
expiresAt: import("drizzle-orm").NotNull<import("drizzle-orm/pg-core").PgTimestampBuilderInitial<string>>;
|
|
172
|
+
};
|
|
173
|
+
//#endregion
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { bigint, bigserial, doublePrecision, integer, jsonb, text, timestamp } from "drizzle-orm/pg-core";
|
|
2
|
+
//#region src/drizzle/index.ts
|
|
3
|
+
/**
|
|
4
|
+
* `octaflow/drizzle` — the engine's tables as Drizzle column sets.
|
|
5
|
+
*
|
|
6
|
+
* One spreadable column set per table the Postgres adapters create: the
|
|
7
|
+
* workflow store's two (`flowStoreDdl`), the event sink's one (`flowEventDdl`)
|
|
8
|
+
* and the gate's two (`flowGateDdl`). A host on Drizzle declares the tables in
|
|
9
|
+
* its own schema from these columns and lets drizzle-kit own the migrations;
|
|
10
|
+
* the stores themselves stay plain SQL over the `schema` (and, for the gate,
|
|
11
|
+
* the table names) the host hands them. Nothing here runs a query: no store
|
|
12
|
+
* is rewritten on the ORM, because every one of the engine's writes is one
|
|
13
|
+
* SQL statement by nature.
|
|
14
|
+
*
|
|
15
|
+
* What the DDL declares beyond the columns is the host's to mirror in its
|
|
16
|
+
* `pgTable(...)` call — the foreign keys, the composite keys, the unique
|
|
17
|
+
* constraint and the indexes. Each set's doc lists them; two of the indexes
|
|
18
|
+
* are load-bearing for the store, not an optimisation, and are marked so.
|
|
19
|
+
* A test holds these sets and the DDL equal column for column.
|
|
20
|
+
*
|
|
21
|
+
* `drizzle-orm` is an optional peer, loaded by this subpath alone.
|
|
22
|
+
*/
|
|
23
|
+
const timestamptz = (name) => timestamp(name, {
|
|
24
|
+
withTimezone: true,
|
|
25
|
+
mode: "date"
|
|
26
|
+
});
|
|
27
|
+
/**
|
|
28
|
+
* `flow_workflow` — one row per run. `partition_key` is the engine's scoping
|
|
29
|
+
* column: the store is bound to one partition and every query filters by it.
|
|
30
|
+
*
|
|
31
|
+
* The DDL adds, and the host declares:
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { index, pgTable, uniqueIndex } from 'drizzle-orm/pg-core';
|
|
35
|
+
* import { sql } from 'drizzle-orm';
|
|
36
|
+
* import { flowWorkflowColumns } from 'octaflow/drizzle';
|
|
37
|
+
*
|
|
38
|
+
* export const flowWorkflow = pgTable('flow_workflow', flowWorkflowColumns, (t) => [
|
|
39
|
+
* index('flow_workflow_partition_status_idx').on(t.partitionKey, t.status),
|
|
40
|
+
* index('flow_workflow_parent_idx').on(t.parentWorkflowId),
|
|
41
|
+
* index('flow_workflow_partition_type_idx').on(t.partitionKey, t.type),
|
|
42
|
+
* index('flow_workflow_partition_entity_idx').on(t.partitionKey, t.entityRef),
|
|
43
|
+
* // Load-bearing: the deadline sweep reads only live runs that have one.
|
|
44
|
+
* index('flow_workflow_deadline_idx').on(t.deadlineAt).where(sql`deadline_at IS NOT NULL`),
|
|
45
|
+
* // Load-bearing: `start` with an idempotency key relies on this unique index
|
|
46
|
+
* // to make a repeated start return the existing run instead of a second one.
|
|
47
|
+
* uniqueIndex('flow_workflow_idempotency_idx').on(t.partitionKey, t.idempotencyKey).where(sql`idempotency_key IS NOT NULL`),
|
|
48
|
+
* ]);
|
|
49
|
+
* ```
|
|
50
|
+
*
|
|
51
|
+
* `parent_workflow_id` and `parent_step_id` carry no foreign key in the DDL
|
|
52
|
+
* (a parent may outlive or predate its child in a migration); add one if you
|
|
53
|
+
* want the constraint.
|
|
54
|
+
*/
|
|
55
|
+
const flowWorkflowColumns = {
|
|
56
|
+
id: bigserial("id", { mode: "number" }).primaryKey(),
|
|
57
|
+
partitionKey: text("partition_key").notNull(),
|
|
58
|
+
/** The workflow definition's name. */
|
|
59
|
+
type: text("type").notNull(),
|
|
60
|
+
status: text("status").notNull(),
|
|
61
|
+
input: jsonb("input").notNull().default({}),
|
|
62
|
+
output: jsonb("output"),
|
|
63
|
+
error: text("error"),
|
|
64
|
+
/** The host's handle for what the run is about (`'record:88'`), for lookups by entity. */
|
|
65
|
+
entityRef: text("entity_ref"),
|
|
66
|
+
totalSteps: integer("total_steps").notNull(),
|
|
67
|
+
completedSteps: integer("completed_steps").notNull().default(0),
|
|
68
|
+
failedSteps: integer("failed_steps").notNull().default(0),
|
|
69
|
+
metadata: jsonb("metadata"),
|
|
70
|
+
/** Set on a `start` that asked for one; unique per partition where present. */
|
|
71
|
+
idempotencyKey: text("idempotency_key"),
|
|
72
|
+
/** The run's deadline, when the definition set one; the sweep reads it. */
|
|
73
|
+
deadlineAt: timestamptz("deadline_at"),
|
|
74
|
+
/** For a sub-workflow: the run that started it, and the step in that run. */
|
|
75
|
+
parentWorkflowId: bigint("parent_workflow_id", { mode: "number" }),
|
|
76
|
+
parentStepId: bigint("parent_step_id", { mode: "number" }),
|
|
77
|
+
createdAt: timestamptz("created_at").notNull().defaultNow(),
|
|
78
|
+
startedAt: timestamptz("started_at"),
|
|
79
|
+
completedAt: timestamptz("completed_at")
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* `flow_workflow_step` — one row per step of a run, including the children a
|
|
83
|
+
* fan-out creates at runtime.
|
|
84
|
+
*
|
|
85
|
+
* The DDL adds, and the host declares:
|
|
86
|
+
*
|
|
87
|
+
* ```ts
|
|
88
|
+
* import { foreignKey, index, pgTable, unique, type AnyPgColumn } from 'drizzle-orm/pg-core';
|
|
89
|
+
* import { flowWorkflowStepColumns } from 'octaflow/drizzle';
|
|
90
|
+
*
|
|
91
|
+
* export const flowWorkflowStep = pgTable('flow_workflow_step', flowWorkflowStepColumns, (t) => [
|
|
92
|
+
* foreignKey({ columns: [t.workflowId], foreignColumns: [flowWorkflow.id] }).onDelete('cascade'),
|
|
93
|
+
* foreignKey({ columns: [t.parentStepId], foreignColumns: [t.id] }).onDelete('cascade'),
|
|
94
|
+
* // `(workflow_id, key)` is the step's identity within a run; the constraint
|
|
95
|
+
* // is what refuses a second step under the same key.
|
|
96
|
+
* unique('flow_workflow_step_workflow_id_key_unique').on(t.workflowId, t.key),
|
|
97
|
+
* index('flow_workflow_step_workflow_idx').on(t.workflowId),
|
|
98
|
+
* index('flow_workflow_step_status_idx').on(t.workflowId, t.status),
|
|
99
|
+
* index('flow_workflow_step_parent_idx').on(t.parentStepId),
|
|
100
|
+
* ]);
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
const flowWorkflowStepColumns = {
|
|
104
|
+
id: bigserial("id", { mode: "number" }).primaryKey(),
|
|
105
|
+
workflowId: bigint("workflow_id", { mode: "number" }).notNull(),
|
|
106
|
+
partitionKey: text("partition_key").notNull(),
|
|
107
|
+
/** The step's key within its workflow — unique per run. */
|
|
108
|
+
key: text("key").notNull(),
|
|
109
|
+
/** The step handler's name. */
|
|
110
|
+
type: text("type").notNull(),
|
|
111
|
+
status: text("status").notNull(),
|
|
112
|
+
/** The keys of the steps this one waits for. */
|
|
113
|
+
dependencies: jsonb("dependencies").notNull().default([]),
|
|
114
|
+
input: jsonb("input"),
|
|
115
|
+
output: jsonb("output"),
|
|
116
|
+
error: text("error"),
|
|
117
|
+
metadata: jsonb("metadata"),
|
|
118
|
+
attempts: integer("attempts").notNull().default(0),
|
|
119
|
+
/** For a fan-out child: the step that spawned it. */
|
|
120
|
+
parentStepId: bigint("parent_step_id", { mode: "number" }),
|
|
121
|
+
/** The last heartbeat a long step sent; the sweep uses it to tell stalled from slow. */
|
|
122
|
+
heartbeatAt: timestamptz("heartbeat_at"),
|
|
123
|
+
startedAt: timestamptz("started_at"),
|
|
124
|
+
completedAt: timestamptz("completed_at")
|
|
125
|
+
};
|
|
126
|
+
/**
|
|
127
|
+
* `flow_step_event` — the append-only run history `createPgEventSink` writes,
|
|
128
|
+
* one row per engine event.
|
|
129
|
+
*
|
|
130
|
+
* The DDL adds, and the host declares:
|
|
131
|
+
*
|
|
132
|
+
* ```ts
|
|
133
|
+
* export const flowStepEvent = pgTable('flow_step_event', flowStepEventColumns, (t) => [
|
|
134
|
+
* index('flow_step_event_workflow_idx').on(t.partitionKey, t.workflowId, t.id),
|
|
135
|
+
* ]);
|
|
136
|
+
* ```
|
|
137
|
+
*
|
|
138
|
+
* `workflow_id` carries no foreign key on purpose: the history outlives the run.
|
|
139
|
+
*/
|
|
140
|
+
const flowStepEventColumns = {
|
|
141
|
+
id: bigserial("id", { mode: "number" }).primaryKey(),
|
|
142
|
+
partitionKey: text("partition_key").notNull(),
|
|
143
|
+
workflowId: bigint("workflow_id", { mode: "number" }).notNull(),
|
|
144
|
+
workflowType: text("workflow_type"),
|
|
145
|
+
/** Null for a workflow-level event. */
|
|
146
|
+
stepId: bigint("step_id", { mode: "number" }),
|
|
147
|
+
stepKey: text("step_key"),
|
|
148
|
+
stepType: text("step_type"),
|
|
149
|
+
eventType: text("event_type").notNull(),
|
|
150
|
+
attempt: integer("attempt"),
|
|
151
|
+
durationMs: integer("duration_ms"),
|
|
152
|
+
error: text("error"),
|
|
153
|
+
at: timestamptz("at").notNull()
|
|
154
|
+
};
|
|
155
|
+
/**
|
|
156
|
+
* `flow_rate_bucket` — one token bucket per `(partition_key, step_type)` for
|
|
157
|
+
* `createPgStepGate`'s rate limits. The table name is the DDL's default and
|
|
158
|
+
* the gate's `tables.rateBucket` option; the two must agree.
|
|
159
|
+
*
|
|
160
|
+
* The DDL adds a composite primary key, and the host declares it:
|
|
161
|
+
*
|
|
162
|
+
* ```ts
|
|
163
|
+
* export const flowRateBucket = pgTable('flow_rate_bucket', flowRateBucketColumns, (t) => [
|
|
164
|
+
* primaryKey({ columns: [t.partitionKey, t.stepType] }),
|
|
165
|
+
* ]);
|
|
166
|
+
* ```
|
|
167
|
+
*/
|
|
168
|
+
const flowRateBucketColumns = {
|
|
169
|
+
partitionKey: text("partition_key").notNull(),
|
|
170
|
+
stepType: text("step_type").notNull(),
|
|
171
|
+
/** The bucket's fill, refilled from `updated_at` on every read. */
|
|
172
|
+
tokens: doublePrecision("tokens").notNull(),
|
|
173
|
+
updatedAt: timestamptz("updated_at").notNull().defaultNow()
|
|
174
|
+
};
|
|
175
|
+
/**
|
|
176
|
+
* `flow_step_lease` — one row per running step under a concurrency cap, for
|
|
177
|
+
* `createPgStepGate`'s `maxConcurrent`. The table name is the DDL's default
|
|
178
|
+
* and the gate's `tables.lease` option; the two must agree.
|
|
179
|
+
*
|
|
180
|
+
* The DDL adds a composite primary key and one index, and the host declares
|
|
181
|
+
* them — the index name is derived from the table name in the DDL:
|
|
182
|
+
*
|
|
183
|
+
* ```ts
|
|
184
|
+
* export const flowStepLease = pgTable('flow_step_lease', flowStepLeaseColumns, (t) => [
|
|
185
|
+
* primaryKey({ columns: [t.partitionKey, t.stepType, t.stepId] }),
|
|
186
|
+
* index('flow_step_lease_active_idx').on(t.partitionKey, t.stepType, t.expiresAt),
|
|
187
|
+
* ]);
|
|
188
|
+
* ```
|
|
189
|
+
*/
|
|
190
|
+
const flowStepLeaseColumns = {
|
|
191
|
+
partitionKey: text("partition_key").notNull(),
|
|
192
|
+
stepType: text("step_type").notNull(),
|
|
193
|
+
stepId: bigint("step_id", { mode: "number" }).notNull(),
|
|
194
|
+
acquiredAt: timestamptz("acquired_at").notNull().defaultNow(),
|
|
195
|
+
/** A lease past this instant is reaped on the next acquire. */
|
|
196
|
+
expiresAt: timestamptz("expires_at").notNull()
|
|
197
|
+
};
|
|
198
|
+
//#endregion
|
|
199
|
+
export { flowRateBucketColumns, flowStepEventColumns, flowStepLeaseColumns, flowWorkflowColumns, flowWorkflowStepColumns };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "octaflow",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.0",
|
|
4
4
|
"description": "Durable DAG workflow engine: Zod-typed steps over Postgres + pg-boss, with an optional AI add-on (token/cost/quota). Layered single package.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -20,6 +20,11 @@
|
|
|
20
20
|
"import": "./dist/postgres/index.js",
|
|
21
21
|
"default": "./dist/postgres/index.js"
|
|
22
22
|
},
|
|
23
|
+
"./drizzle": {
|
|
24
|
+
"types": "./dist/drizzle/index.d.ts",
|
|
25
|
+
"import": "./dist/drizzle/index.js",
|
|
26
|
+
"default": "./dist/drizzle/index.js"
|
|
27
|
+
},
|
|
23
28
|
"./pg-boss": {
|
|
24
29
|
"types": "./dist/pg-boss/index.d.ts",
|
|
25
30
|
"import": "./dist/pg-boss/index.js",
|
|
@@ -65,7 +70,8 @@
|
|
|
65
70
|
"llm",
|
|
66
71
|
"pg-boss",
|
|
67
72
|
"postgres",
|
|
68
|
-
"zod"
|
|
73
|
+
"zod",
|
|
74
|
+
"drizzle"
|
|
69
75
|
],
|
|
70
76
|
"peerDependencies": {
|
|
71
77
|
"@ai-sdk/anthropic": "^4.0.0",
|
|
@@ -74,6 +80,7 @@
|
|
|
74
80
|
"@ai-sdk/openai-compatible": "^3.0.0",
|
|
75
81
|
"@ai-sdk/provider": "^4.0.0",
|
|
76
82
|
"ai": "^7.0.0",
|
|
83
|
+
"drizzle-orm": ">=0.36.0 <1",
|
|
77
84
|
"pg": "^8.0.0",
|
|
78
85
|
"pg-boss": "^12.21.0",
|
|
79
86
|
"typescript": "^5 || ^6 || ^7",
|
|
@@ -98,6 +105,9 @@
|
|
|
98
105
|
"ai": {
|
|
99
106
|
"optional": true
|
|
100
107
|
},
|
|
108
|
+
"drizzle-orm": {
|
|
109
|
+
"optional": true
|
|
110
|
+
},
|
|
101
111
|
"pg": {
|
|
102
112
|
"optional": true
|
|
103
113
|
},
|
|
@@ -115,6 +125,7 @@
|
|
|
115
125
|
"@types/node": "^26.5.1",
|
|
116
126
|
"@types/pg": "^8.23.1",
|
|
117
127
|
"ai": "^7.0.97",
|
|
128
|
+
"drizzle-orm": "^0.45.2",
|
|
118
129
|
"pg": "^8.23.0",
|
|
119
130
|
"pg-boss": "^12.30.0",
|
|
120
131
|
"testcontainers": "^12.1.0",
|