@pikku/kysely 0.13.2 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/kysely-ai-run-state-service.js +3 -18
- package/dist/src/kysely-ai-storage-service.d.ts +0 -1
- package/dist/src/kysely-ai-storage-service.js +4 -101
- package/dist/src/kysely-channel-store.js +4 -24
- package/dist/src/kysely-credential-service.js +3 -24
- package/dist/src/kysely-deployment-service.d.ts +0 -1
- package/dist/src/kysely-deployment-service.js +4 -48
- package/dist/src/kysely-scope-service.js +4 -54
- package/dist/src/kysely-secret-service.js +3 -21
- package/dist/src/kysely-session-store.js +4 -11
- package/dist/src/kysely-tables.d.ts +2 -0
- package/dist/src/kysely-webhook-service.d.ts +0 -1
- package/dist/src/kysely-webhook-service.js +4 -51
- package/dist/src/kysely-workflow-mirror.d.ts +7 -3
- package/dist/src/kysely-workflow-mirror.js +10 -87
- package/dist/src/kysely-workflow-service.d.ts +2 -2
- package/dist/src/kysely-workflow-service.js +11 -95
- package/dist/src/schema/ai.schema.d.ts +12 -0
- package/dist/src/schema/ai.schema.js +89 -0
- package/dist/src/schema/channel.schema.d.ts +3 -0
- package/dist/src/schema/channel.schema.js +24 -0
- package/dist/src/schema/credential.schema.d.ts +11 -0
- package/dist/src/schema/credential.schema.js +33 -0
- package/dist/src/schema/deployment.schema.d.ts +3 -0
- package/dist/src/schema/deployment.schema.js +32 -0
- package/dist/src/schema/index.d.ts +82 -0
- package/dist/src/schema/index.js +187 -0
- package/dist/src/schema/pikku-schema.types.d.ts +75 -0
- package/dist/src/schema/pikku-schema.types.js +21 -0
- package/dist/src/schema/scope.schema.d.ts +18 -0
- package/dist/src/schema/scope.schema.js +61 -0
- package/dist/src/schema/secret.schema.d.ts +11 -0
- package/dist/src/schema/secret.schema.js +29 -0
- package/dist/src/schema/session.schema.d.ts +3 -0
- package/dist/src/schema/session.schema.js +13 -0
- package/dist/src/schema/webhook.schema.d.ts +3 -0
- package/dist/src/schema/webhook.schema.js +38 -0
- package/dist/src/schema/workflow.schema.d.ts +15 -0
- package/dist/src/schema/workflow.schema.js +86 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/index.ts +18 -0
- package/src/kysely-ai-run-state-service.ts +7 -28
- package/src/kysely-ai-storage-service.ts +4 -152
- package/src/kysely-channel-store.ts +4 -34
- package/src/kysely-credential-service.ts +3 -36
- package/src/kysely-deployment-service.ts +4 -63
- package/src/kysely-scope-service.ts +4 -81
- package/src/kysely-secret-service.ts +6 -33
- package/src/kysely-session-store.ts +4 -18
- package/src/kysely-tables.test.ts +110 -0
- package/src/kysely-tables.ts +2 -0
- package/src/kysely-webhook-service.ts +4 -67
- package/src/kysely-workflow-mirror.test.ts +7 -7
- package/src/kysely-workflow-mirror.ts +10 -109
- package/src/kysely-workflow-service.ts +18 -125
- package/src/schema/ai.schema.ts +140 -0
- package/src/schema/channel.schema.ts +37 -0
- package/src/schema/credential.schema.ts +46 -0
- package/src/schema/deployment.schema.ts +47 -0
- package/src/schema/index.ts +249 -0
- package/src/schema/pikku-schema.types.ts +94 -0
- package/src/schema/schema.test.ts +273 -0
- package/src/schema/scope.schema.ts +95 -0
- package/src/schema/secret.schema.ts +40 -0
- package/src/schema/session.schema.ts +20 -0
- package/src/schema/webhook.schema.ts +57 -0
- package/src/schema/workflow.schema.ts +113 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,59 @@
|
|
|
1
|
+
## 0.13.4
|
|
2
|
+
|
|
3
|
+
### Patch Changes
|
|
4
|
+
|
|
5
|
+
- 91077ff: Declare the pikku runtime tables instead of creating them at boot.
|
|
6
|
+
|
|
7
|
+
The tables the runtime needs — workflow, AI, scopes, secrets, credentials, webhooks, channels, deployments, sessions — were created by `createTable(...).ifNotExists()` inside each service's `init()`. A table therefore existed if and only if some service happened to be constructed, in whatever order, with whatever schema qualifier that call site passed. In one deployment that produced a duplicate set of AI tables in `public` shadowing the intended ones in `app`.
|
|
8
|
+
|
|
9
|
+
`pikkuSchemas` is now the single declaration, exported from `@pikku/kysely`: the same kysely schema-builder chains, moved out of `init()` and never executed at boot. `applyPikkuSchemas` materializes them onto a database, `compilePikkuSchemas` renders them as SQL for a dialect, and both bind `CamelCasePlugin` themselves so a declaration always compiles to the physical names the rest of the package queries.
|
|
10
|
+
|
|
11
|
+
`applyPikkuSchemas` runs its statements in one transaction, so the promise it makes about a half-applied database holds for the failures the prerequisite check cannot foresee too — a statement colliding with a table something else already created rolls back the ones that ran before it.
|
|
12
|
+
|
|
13
|
+
Auth is a declared prerequisite. `pikkuUserRole` and `pikkuUserScope` reference `user.id`, which Better Auth owns, so `scopeSchema` says so via `requires` and `applyPikkuSchemas` checks every prerequisite before creating anything — a project without auth gets a sentence naming what is missing and who owns it, not a foreign key error and not a half-applied database. `resolveRequirements` returns the same answer as data for callers that need to describe the gap rather than die on it.
|
|
14
|
+
|
|
15
|
+
That prerequisite also fixes a fourth disagreement: `user_id` was declared `text`, but `user.id` has no fixed type — Better Auth decides it from config, giving `uuid` under `generateId: 'uuid'` and an identity `integer` under `'serial'`, and postgres refuses a `text` column referencing either. The whole statement was rejected, so `KyselyScopeService.init()` had never actually created these tables on such a project — one had a migration written by hand with a comment explaining why. The column now takes its type from the column it references, whatever that turns out to be.
|
|
16
|
+
|
|
17
|
+
Three further disagreements the single declaration settled:
|
|
18
|
+
- `aiRun` was created by two services with different columns. `KyselyAIStorageService` omitted `pendingApprovals`; `KyselyAIRunStateService` declared it and read it back through `as any` casts. Whichever ran first decided the shape, and where the storage service won, resolving an approval failed. The column is declared.
|
|
19
|
+
- `workflowRuns`, `workflowStep` and `workflowStepHistory` defaulted their primary keys to `sql.raw("'" + crypto.randomUUID() + "'")`, evaluated once when the statement was built — one shared default for every row, not a generator. Dropped; the services supply the id.
|
|
20
|
+
- `workflowStep.fromStepName` was backfilled by an `alterTable(...).execute().catch(() => {})` bolted on after the fact. It is part of the declaration.
|
|
21
|
+
|
|
22
|
+
Every service's `init()` now goes through `ensurePikkuSchema`, which looks before it creates: all the tables present is a no-op, none present creates them from the declaration, and some present throws rather than filling in the rest. That last case is the point. `.ifNotExists()` turned every failure into a silent success — it is why nobody noticed the `user_id` type had been rejected since the day it was written — and a schema half-owned by a migration and half by a boot-time create is the condition all of this exists to end. Around 615 lines of duplicated DDL went with it.
|
|
23
|
+
|
|
24
|
+
Creating at boot is now the fallback rather than the intent: run `pikku db generate` and `init()` takes the `present` path and issues no DDL at all.
|
|
25
|
+
|
|
26
|
+
`KyselyPikkuDB` stays hand-written, because it carries what introspection cannot recover — `WorkflowStatus` rather than `string`, `Generated<Date>` rather than `Date`. A test materializes the declaration and asserts the two agree on every table and column, which is the drift protection generating it would have bought. It found one on its first run: `ai_run.pending_approvals` was in the DDL and in `KyselyAIRunStateService`, but not in `AIRunTable` — the reason that service read it back through `as any`. Declared, and the casts are gone.
|
|
27
|
+
|
|
28
|
+
## 0.13.3
|
|
29
|
+
|
|
30
|
+
### Patch Changes
|
|
31
|
+
|
|
32
|
+
- e3d4454: Add job groups, so one shared queue can stay fair without splitting into one
|
|
33
|
+
queue per producer.
|
|
34
|
+
|
|
35
|
+
A job may now carry `group: { id, tier }`, and a worker may cap how many jobs
|
|
36
|
+
of any one group run at once via `groupConcurrency`. On pg-boss this maps to
|
|
37
|
+
`localGroupConcurrency`, which excludes at-capacity groups from the fetch query
|
|
38
|
+
itself, so a capped group costs nothing rather than being fetched and restored.
|
|
39
|
+
BullMQ declares it unsupported (groups are a BullMQ Pro feature) — being
|
|
40
|
+
push-based, it can simply use a queue per group at no polling cost.
|
|
41
|
+
|
|
42
|
+
Workflow services accept a `queueStrategy`. The default `'per-workflow'` is
|
|
43
|
+
unchanged: every workflow gets its own `wf-orchestrator-*` / `wf-step-*` queue,
|
|
44
|
+
which is also what lets serverless providers deploy one unit per workflow. The
|
|
45
|
+
new `'shared-groups'` routes every workflow through the shared
|
|
46
|
+
orchestrator/step-worker queues and isolates them by group instead, so a
|
|
47
|
+
monolith runs one set of pollers rather than one per workflow — on a
|
|
48
|
+
pull-based backend with dozens of workflows that is the difference between
|
|
49
|
+
hundreds of poll loops and twenty. It is for single-process runtimes only; a
|
|
50
|
+
per-unit serverless deploy still needs the per-workflow queues to route to its
|
|
51
|
+
units.
|
|
52
|
+
|
|
53
|
+
- Updated dependencies [24252b8]
|
|
54
|
+
- Updated dependencies [e3d4454]
|
|
55
|
+
- @pikku/core@0.12.69
|
|
56
|
+
|
|
1
57
|
## 0.13.2
|
|
2
58
|
|
|
3
59
|
### Patch Changes
|
package/dist/src/index.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export { KyselyWebhookService } from './kysely-webhook-service.js';
|
|
|
17
17
|
export { createAuditedKysely, type CreateAuditedKyselyOptions, } from './create-audited-kysely.js';
|
|
18
18
|
export { KyselyAuditService } from './kysely-audit-service.js';
|
|
19
19
|
export { SerializePlugin, SqliteSerializePlugin, BaseSerializePlugin, defaultSerializer, defaultDeserializer, maybeJson, skipTransform, dateRegex, type Serializer, type Deserializer, type SerializePluginOptions, } from './serialize-plugin.js';
|
|
20
|
+
export { pikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, ensurePikkuSchema, type EnsureOutcome, resolveRequirements, unsatisfiedRequirements, rawStatement, requiredType, type PikkuSchema, type RequiredTypes, type SchemaRequirement, type UnmetRequirement, type SchemaStatement, type SchemaStatementFactory, } from './schema/index.js';
|
|
20
21
|
export type { KyselyPikkuDB } from './kysely-tables.js';
|
|
21
22
|
export type { WorkflowRunService } from '@pikku/core/workflow';
|
|
22
23
|
export type { AgentRunService, AgentRunRow } from '@pikku/core/ai-agent';
|
package/dist/src/index.js
CHANGED
|
@@ -15,3 +15,4 @@ export { KyselyWebhookService } from './kysely-webhook-service.js';
|
|
|
15
15
|
export { createAuditedKysely, } from './create-audited-kysely.js';
|
|
16
16
|
export { KyselyAuditService } from './kysely-audit-service.js';
|
|
17
17
|
export { SerializePlugin, SqliteSerializePlugin, BaseSerializePlugin, defaultSerializer, defaultDeserializer, maybeJson, skipTransform, dateRegex, } from './serialize-plugin.js';
|
|
18
|
+
export { pikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, ensurePikkuSchema, resolveRequirements, unsatisfiedRequirements, rawStatement, requiredType, } from './schema/index.js';
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { sql } from 'kysely';
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { aiSchema } from './schema/ai.schema.js';
|
|
2
4
|
export class KyselyAIRunStateService {
|
|
3
5
|
db;
|
|
4
6
|
initialized = false;
|
|
@@ -8,24 +10,7 @@ export class KyselyAIRunStateService {
|
|
|
8
10
|
async init() {
|
|
9
11
|
if (this.initialized)
|
|
10
12
|
return;
|
|
11
|
-
await this.db
|
|
12
|
-
.createTable('aiRun')
|
|
13
|
-
.ifNotExists()
|
|
14
|
-
.addColumn('runId', 'text', (col) => col.primaryKey())
|
|
15
|
-
.addColumn('agentName', 'text', (col) => col.notNull())
|
|
16
|
-
.addColumn('threadId', 'text', (col) => col.notNull())
|
|
17
|
-
.addColumn('resourceId', 'text', (col) => col.notNull())
|
|
18
|
-
.addColumn('status', 'text', (col) => col.defaultTo('running').notNull())
|
|
19
|
-
.addColumn('errorMessage', 'text')
|
|
20
|
-
.addColumn('suspendReason', 'text')
|
|
21
|
-
.addColumn('missingRpcs', 'text')
|
|
22
|
-
.addColumn('pendingApprovals', 'text')
|
|
23
|
-
.addColumn('usageInputTokens', 'integer', (col) => col.defaultTo(0))
|
|
24
|
-
.addColumn('usageOutputTokens', 'integer', (col) => col.defaultTo(0))
|
|
25
|
-
.addColumn('usageModel', 'text', (col) => col.defaultTo(''))
|
|
26
|
-
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
27
|
-
.addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
28
|
-
.execute();
|
|
13
|
+
await ensurePikkuSchema(this.db, aiSchema);
|
|
29
14
|
this.initialized = true;
|
|
30
15
|
}
|
|
31
16
|
async createRun(run) {
|
|
@@ -6,7 +6,6 @@ export declare class KyselyAIStorageService implements AIStorageService, AIRunSt
|
|
|
6
6
|
private db;
|
|
7
7
|
private initialized;
|
|
8
8
|
constructor(db: Kysely<KyselyPikkuDB>);
|
|
9
|
-
private createIndexSafe;
|
|
10
9
|
init(): Promise<void>;
|
|
11
10
|
createThread(resourceId: string, options?: {
|
|
12
11
|
threadId?: string;
|
|
@@ -1,113 +1,16 @@
|
|
|
1
|
-
import { sql } from 'kysely';
|
|
2
1
|
import { parseJson } from './kysely-json.js';
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { aiSchema } from './schema/ai.schema.js';
|
|
3
4
|
export class KyselyAIStorageService {
|
|
4
5
|
db;
|
|
5
6
|
initialized = false;
|
|
6
7
|
constructor(db) {
|
|
7
8
|
this.db = db;
|
|
8
9
|
}
|
|
9
|
-
async createIndexSafe(builder) {
|
|
10
|
-
try {
|
|
11
|
-
await builder.execute();
|
|
12
|
-
}
|
|
13
|
-
catch (e) {
|
|
14
|
-
// Ignore "index already exists" errors across databases
|
|
15
|
-
// MySQL: ER_DUP_KEYNAME, Postgres: 42P07, SQLite: "already exists"
|
|
16
|
-
if (e?.code === 'ER_DUP_KEYNAME' || e?.errno === 1061)
|
|
17
|
-
return;
|
|
18
|
-
if (e?.code === '42P07')
|
|
19
|
-
return;
|
|
20
|
-
if (e?.message?.includes('already exists'))
|
|
21
|
-
return;
|
|
22
|
-
throw e;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
10
|
async init() {
|
|
26
|
-
if (this.initialized)
|
|
11
|
+
if (this.initialized)
|
|
27
12
|
return;
|
|
28
|
-
|
|
29
|
-
await this.db.schema
|
|
30
|
-
.createTable('ai_threads')
|
|
31
|
-
.ifNotExists()
|
|
32
|
-
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
33
|
-
.addColumn('resource_id', 'varchar(255)', (col) => col.notNull())
|
|
34
|
-
.addColumn('title', 'text')
|
|
35
|
-
.addColumn('metadata', 'text')
|
|
36
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
37
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
38
|
-
.execute();
|
|
39
|
-
await this.createIndexSafe(this.db.schema
|
|
40
|
-
.createIndex('idx_ai_threads_resource')
|
|
41
|
-
.on('ai_threads')
|
|
42
|
-
.column('resource_id'));
|
|
43
|
-
await this.db.schema
|
|
44
|
-
.createTable('ai_message')
|
|
45
|
-
.ifNotExists()
|
|
46
|
-
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
47
|
-
.addColumn('thread_id', 'varchar(36)', (col) => col.notNull().references('ai_threads.id').onDelete('cascade'))
|
|
48
|
-
.addColumn('role', 'varchar(50)', (col) => col.notNull())
|
|
49
|
-
.addColumn('content', 'text')
|
|
50
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
51
|
-
.execute();
|
|
52
|
-
await this.createIndexSafe(this.db.schema
|
|
53
|
-
.createIndex('idx_ai_message_thread')
|
|
54
|
-
.on('ai_message')
|
|
55
|
-
.columns(['thread_id', 'created_at']));
|
|
56
|
-
await this.db.schema
|
|
57
|
-
.createTable('ai_tool_call')
|
|
58
|
-
.ifNotExists()
|
|
59
|
-
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
60
|
-
.addColumn('thread_id', 'varchar(36)', (col) => col.notNull().references('ai_threads.id').onDelete('cascade'))
|
|
61
|
-
.addColumn('message_id', 'varchar(36)', (col) => col.notNull().references('ai_message.id').onDelete('cascade'))
|
|
62
|
-
.addColumn('run_id', 'varchar(36)')
|
|
63
|
-
.addColumn('tool_name', 'varchar(255)', (col) => col.notNull())
|
|
64
|
-
.addColumn('args', 'text', (col) => col.notNull())
|
|
65
|
-
.addColumn('result', 'text')
|
|
66
|
-
.addColumn('approval_status', 'varchar(50)')
|
|
67
|
-
.addColumn('approval_type', 'varchar(50)')
|
|
68
|
-
.addColumn('agent_run_id', 'varchar(36)')
|
|
69
|
-
.addColumn('display_tool_name', 'varchar(255)')
|
|
70
|
-
.addColumn('display_args', 'text')
|
|
71
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
72
|
-
.execute();
|
|
73
|
-
await this.createIndexSafe(this.db.schema
|
|
74
|
-
.createIndex('idx_ai_tool_call_thread')
|
|
75
|
-
.on('ai_tool_call')
|
|
76
|
-
.column('thread_id'));
|
|
77
|
-
await this.createIndexSafe(this.db.schema
|
|
78
|
-
.createIndex('idx_ai_tool_call_message')
|
|
79
|
-
.on('ai_tool_call')
|
|
80
|
-
.column('message_id'));
|
|
81
|
-
await this.db.schema
|
|
82
|
-
.createTable('ai_working_memory')
|
|
83
|
-
.ifNotExists()
|
|
84
|
-
.addColumn('id', 'varchar(255)', (col) => col.notNull())
|
|
85
|
-
.addColumn('scope', 'varchar(50)', (col) => col.notNull())
|
|
86
|
-
.addColumn('data', 'text', (col) => col.notNull())
|
|
87
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
88
|
-
.addPrimaryKeyConstraint('ai_working_memory_pk', ['id', 'scope'])
|
|
89
|
-
.execute();
|
|
90
|
-
await this.db.schema
|
|
91
|
-
.createTable('ai_run')
|
|
92
|
-
.ifNotExists()
|
|
93
|
-
.addColumn('run_id', 'varchar(36)', (col) => col.primaryKey())
|
|
94
|
-
.addColumn('agent_name', 'varchar(255)', (col) => col.notNull())
|
|
95
|
-
.addColumn('thread_id', 'varchar(36)', (col) => col.notNull().references('ai_threads.id').onDelete('cascade'))
|
|
96
|
-
.addColumn('resource_id', 'varchar(255)', (col) => col.notNull())
|
|
97
|
-
.addColumn('status', 'varchar(50)', (col) => col.notNull().defaultTo('running'))
|
|
98
|
-
.addColumn('error_message', 'text')
|
|
99
|
-
.addColumn('suspend_reason', 'text')
|
|
100
|
-
.addColumn('missing_rpcs', 'text')
|
|
101
|
-
.addColumn('usage_input_tokens', 'integer', (col) => col.notNull().defaultTo(0))
|
|
102
|
-
.addColumn('usage_output_tokens', 'integer', (col) => col.notNull().defaultTo(0))
|
|
103
|
-
.addColumn('usage_model', 'varchar(255)', (col) => col.notNull().defaultTo(''))
|
|
104
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
105
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
106
|
-
.execute();
|
|
107
|
-
await this.createIndexSafe(this.db.schema
|
|
108
|
-
.createIndex('idx_ai_run_thread')
|
|
109
|
-
.on('ai_run')
|
|
110
|
-
.columns(['thread_id', 'created_at']));
|
|
13
|
+
await ensurePikkuSchema(this.db, aiSchema);
|
|
111
14
|
this.initialized = true;
|
|
112
15
|
}
|
|
113
16
|
async createThread(resourceId, options) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ChannelStore } from '@pikku/core/channel';
|
|
2
|
-
import { sql } from 'kysely';
|
|
3
2
|
import { parseJson } from './kysely-json.js';
|
|
3
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
4
|
+
import { channelSchema } from './schema/channel.schema.js';
|
|
4
5
|
export class KyselyChannelStore extends ChannelStore {
|
|
5
6
|
db;
|
|
6
7
|
initialized = false;
|
|
@@ -9,30 +10,9 @@ export class KyselyChannelStore extends ChannelStore {
|
|
|
9
10
|
this.db = db;
|
|
10
11
|
}
|
|
11
12
|
async init() {
|
|
12
|
-
if (this.initialized)
|
|
13
|
+
if (this.initialized)
|
|
13
14
|
return;
|
|
14
|
-
|
|
15
|
-
await this.db.schema
|
|
16
|
-
.createTable('channels')
|
|
17
|
-
.ifNotExists()
|
|
18
|
-
.addColumn('channel_id', 'text', (col) => col.primaryKey())
|
|
19
|
-
.addColumn('channel_name', 'text', (col) => col.notNull())
|
|
20
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
21
|
-
.addColumn('opening_data', 'text', (col) => col.notNull().defaultTo('{}'))
|
|
22
|
-
.addColumn('pikku_user_id', 'text')
|
|
23
|
-
.addColumn('state', 'text')
|
|
24
|
-
.addColumn('last_wire', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
25
|
-
.execute();
|
|
26
|
-
await this.db.schema
|
|
27
|
-
.createTable('channel_subscriptions')
|
|
28
|
-
.ifNotExists()
|
|
29
|
-
.addColumn('channel_id', 'text', (col) => col.notNull().references('channels.channel_id').onDelete('cascade'))
|
|
30
|
-
.addColumn('topic', 'text', (col) => col.notNull())
|
|
31
|
-
.addPrimaryKeyConstraint('channel_subscriptions_pk', [
|
|
32
|
-
'channel_id',
|
|
33
|
-
'topic',
|
|
34
|
-
])
|
|
35
|
-
.execute();
|
|
15
|
+
await ensurePikkuSchema(this.db, channelSchema);
|
|
36
16
|
this.initialized = true;
|
|
37
17
|
}
|
|
38
18
|
async addChannel({ channelId, channelName, openingData, }) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { sql } from 'kysely';
|
|
2
1
|
import { envelopeEncrypt, envelopeDecrypt, envelopeRewrap, } from '@pikku/core/crypto-utils';
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { credentialSchema } from './schema/credential.schema.js';
|
|
3
4
|
export class KyselyCredentialService {
|
|
4
5
|
db;
|
|
5
6
|
initialized = false;
|
|
@@ -19,29 +20,7 @@ export class KyselyCredentialService {
|
|
|
19
20
|
async init() {
|
|
20
21
|
if (this.initialized)
|
|
21
22
|
return;
|
|
22
|
-
await this.db
|
|
23
|
-
.createTable('credentials')
|
|
24
|
-
.ifNotExists()
|
|
25
|
-
.addColumn('name', 'varchar(255)', (col) => col.notNull())
|
|
26
|
-
.addColumn('user_id', 'varchar(255)')
|
|
27
|
-
.addColumn('ciphertext', 'text', (col) => col.notNull())
|
|
28
|
-
.addColumn('wrapped_dek', 'text', (col) => col.notNull())
|
|
29
|
-
.addColumn('key_version', 'integer', (col) => col.notNull())
|
|
30
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
31
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
32
|
-
.execute();
|
|
33
|
-
await sql `CREATE UNIQUE INDEX IF NOT EXISTS credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`.execute(this.db);
|
|
34
|
-
if (this.audit) {
|
|
35
|
-
await this.db.schema
|
|
36
|
-
.createTable('credentials_audit')
|
|
37
|
-
.ifNotExists()
|
|
38
|
-
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
39
|
-
.addColumn('credential_name', 'varchar(255)', (col) => col.notNull())
|
|
40
|
-
.addColumn('user_id', 'varchar(255)')
|
|
41
|
-
.addColumn('action', 'varchar(20)', (col) => col.notNull())
|
|
42
|
-
.addColumn('performed_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
43
|
-
.execute();
|
|
44
|
-
}
|
|
23
|
+
await ensurePikkuSchema(this.db, credentialSchema);
|
|
45
24
|
this.initialized = true;
|
|
46
25
|
}
|
|
47
26
|
async logAudit(name, userId, action) {
|
|
@@ -16,6 +16,5 @@ export declare class KyselyDeploymentService implements DeploymentService {
|
|
|
16
16
|
start(config: DeploymentConfig): Promise<void>;
|
|
17
17
|
stop(): Promise<void>;
|
|
18
18
|
invoke(funcName: string, data: unknown, session?: unknown, traceId?: string): Promise<unknown>;
|
|
19
|
-
private createIndexSafe;
|
|
20
19
|
private sendHeartbeat;
|
|
21
20
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { buildRemoteHeaders } from '@pikku/core/remote';
|
|
2
2
|
import { getAllFunctionNames } from '@pikku/core/function';
|
|
3
|
-
import {
|
|
3
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
4
|
+
import { deploymentSchema } from './schema/deployment.schema.js';
|
|
4
5
|
export class KyselyDeploymentService {
|
|
5
6
|
db;
|
|
6
7
|
jwt;
|
|
@@ -18,40 +19,9 @@ export class KyselyDeploymentService {
|
|
|
18
19
|
this.heartbeatTtl = config.heartbeatTtl ?? 30000;
|
|
19
20
|
}
|
|
20
21
|
async init() {
|
|
21
|
-
if (this.initialized)
|
|
22
|
+
if (this.initialized)
|
|
22
23
|
return;
|
|
23
|
-
|
|
24
|
-
await this.db.schema
|
|
25
|
-
.createTable('pikku_deployments')
|
|
26
|
-
.ifNotExists()
|
|
27
|
-
.addColumn('deployment_id', 'text', (col) => col.primaryKey())
|
|
28
|
-
.addColumn('endpoint', 'text', (col) => col.notNull())
|
|
29
|
-
.addColumn('last_heartbeat', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
30
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
31
|
-
.execute();
|
|
32
|
-
await this.db.schema
|
|
33
|
-
.createTable('pikku_deployment_functions')
|
|
34
|
-
.ifNotExists()
|
|
35
|
-
.addColumn('deployment_id', 'text', (col) => col
|
|
36
|
-
.notNull()
|
|
37
|
-
.references('pikku_deployments.deployment_id')
|
|
38
|
-
.onDelete('cascade'))
|
|
39
|
-
.addColumn('function_name', 'text', (col) => col.notNull())
|
|
40
|
-
.addPrimaryKeyConstraint('pikku_deployment_functions_pk', [
|
|
41
|
-
'deployment_id',
|
|
42
|
-
'function_name',
|
|
43
|
-
])
|
|
44
|
-
.execute();
|
|
45
|
-
await this.createIndexSafe(this.db.schema
|
|
46
|
-
.createIndex('idx_pikku_deployments_heartbeat')
|
|
47
|
-
.ifNotExists()
|
|
48
|
-
.on('pikku_deployments')
|
|
49
|
-
.column('last_heartbeat'));
|
|
50
|
-
await this.createIndexSafe(this.db.schema
|
|
51
|
-
.createIndex('idx_pikku_deployment_functions_name')
|
|
52
|
-
.ifNotExists()
|
|
53
|
-
.on('pikku_deployment_functions')
|
|
54
|
-
.column('function_name'));
|
|
24
|
+
await ensurePikkuSchema(this.db, deploymentSchema);
|
|
55
25
|
this.initialized = true;
|
|
56
26
|
}
|
|
57
27
|
async start(config) {
|
|
@@ -129,20 +99,6 @@ export class KyselyDeploymentService {
|
|
|
129
99
|
}
|
|
130
100
|
return response.json();
|
|
131
101
|
}
|
|
132
|
-
async createIndexSafe(builder) {
|
|
133
|
-
try {
|
|
134
|
-
await builder.execute();
|
|
135
|
-
}
|
|
136
|
-
catch (e) {
|
|
137
|
-
if (e?.code === 'ER_DUP_KEYNAME' || e?.errno === 1061)
|
|
138
|
-
return;
|
|
139
|
-
if (e?.code === '42P07')
|
|
140
|
-
return;
|
|
141
|
-
if (e?.message?.includes('already exists'))
|
|
142
|
-
return;
|
|
143
|
-
throw e;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
102
|
async sendHeartbeat() {
|
|
147
103
|
if (!this.deploymentConfig)
|
|
148
104
|
return;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
2
|
+
import { scopeSchema } from './schema/scope.schema.js';
|
|
2
3
|
/**
|
|
3
4
|
* Resolves and administers user scopes against four self-created tables.
|
|
4
5
|
*
|
|
@@ -18,60 +19,9 @@ export class KyselyScopeService {
|
|
|
18
19
|
this.db = db;
|
|
19
20
|
}
|
|
20
21
|
async init() {
|
|
21
|
-
if (this.initialized)
|
|
22
|
+
if (this.initialized)
|
|
22
23
|
return;
|
|
23
|
-
|
|
24
|
-
await this.db.schema
|
|
25
|
-
.createTable('pikku_scopes')
|
|
26
|
-
.ifNotExists()
|
|
27
|
-
.addColumn('name', 'text', (col) => col.primaryKey())
|
|
28
|
-
.addColumn('description', 'text')
|
|
29
|
-
.addColumn('declared', 'boolean', (col) => col.defaultTo(true).notNull())
|
|
30
|
-
.execute();
|
|
31
|
-
await this.db.schema
|
|
32
|
-
.createTable('pikku_roles')
|
|
33
|
-
.ifNotExists()
|
|
34
|
-
.addColumn('name', 'text', (col) => col.primaryKey())
|
|
35
|
-
.addColumn('description', 'text')
|
|
36
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
37
|
-
.execute();
|
|
38
|
-
await this.db.schema
|
|
39
|
-
.createTable('pikku_role_scopes')
|
|
40
|
-
.ifNotExists()
|
|
41
|
-
.addColumn('role', 'text', (col) => col.notNull().references('pikku_roles.name').onDelete('cascade'))
|
|
42
|
-
.addColumn('scope', 'text', (col) => col.notNull().references('pikku_scopes.name').onDelete('cascade'))
|
|
43
|
-
.addPrimaryKeyConstraint('pikku_role_scopes_pk', ['role', 'scope'])
|
|
44
|
-
.execute();
|
|
45
|
-
await this.db.schema
|
|
46
|
-
.createTable('pikku_user_role')
|
|
47
|
-
.ifNotExists()
|
|
48
|
-
.addColumn('user_id', 'text', (col) => col.notNull().references('user.id').onDelete('cascade'))
|
|
49
|
-
.addColumn('role', 'text', (col) => col.notNull().references('pikku_roles.name').onDelete('cascade'))
|
|
50
|
-
.addColumn('granted_by', 'text')
|
|
51
|
-
.addColumn('granted_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
52
|
-
.addPrimaryKeyConstraint('pikku_user_role_pk', ['user_id', 'role'])
|
|
53
|
-
.execute();
|
|
54
|
-
await this.db.schema
|
|
55
|
-
.createTable('pikku_user_scope')
|
|
56
|
-
.ifNotExists()
|
|
57
|
-
.addColumn('user_id', 'text', (col) => col.notNull().references('user.id').onDelete('cascade'))
|
|
58
|
-
.addColumn('scope', 'text', (col) => col.notNull().references('pikku_scopes.name').onDelete('cascade'))
|
|
59
|
-
.addColumn('granted_by', 'text')
|
|
60
|
-
.addColumn('granted_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
61
|
-
.addPrimaryKeyConstraint('pikku_user_scope_pk', ['user_id', 'scope'])
|
|
62
|
-
.execute();
|
|
63
|
-
await this.db.schema
|
|
64
|
-
.createIndex('pikku_role_scopes_scope_idx')
|
|
65
|
-
.ifNotExists()
|
|
66
|
-
.on('pikku_role_scopes')
|
|
67
|
-
.column('scope')
|
|
68
|
-
.execute();
|
|
69
|
-
await this.db.schema
|
|
70
|
-
.createIndex('pikku_user_role_role_idx')
|
|
71
|
-
.ifNotExists()
|
|
72
|
-
.on('pikku_user_role')
|
|
73
|
-
.column('role')
|
|
74
|
-
.execute();
|
|
24
|
+
await ensurePikkuSchema(this.db, scopeSchema);
|
|
75
25
|
this.initialized = true;
|
|
76
26
|
}
|
|
77
27
|
/**
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { sql } from 'kysely';
|
|
2
1
|
import { envelopeEncrypt, envelopeDecrypt, envelopeRewrap, } from '@pikku/core/crypto-utils';
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { secretSchema } from './schema/secret.schema.js';
|
|
3
4
|
export class KyselySecretService {
|
|
4
5
|
db;
|
|
5
6
|
initialized = false;
|
|
@@ -19,26 +20,7 @@ export class KyselySecretService {
|
|
|
19
20
|
async init() {
|
|
20
21
|
if (this.initialized)
|
|
21
22
|
return;
|
|
22
|
-
await this.db
|
|
23
|
-
.createTable('secrets')
|
|
24
|
-
.ifNotExists()
|
|
25
|
-
.addColumn('key', 'varchar(255)', (col) => col.primaryKey())
|
|
26
|
-
.addColumn('ciphertext', 'text', (col) => col.notNull())
|
|
27
|
-
.addColumn('wrapped_dek', 'text', (col) => col.notNull())
|
|
28
|
-
.addColumn('key_version', 'integer', (col) => col.notNull())
|
|
29
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
30
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
31
|
-
.execute();
|
|
32
|
-
if (this.audit) {
|
|
33
|
-
await this.db.schema
|
|
34
|
-
.createTable('secrets_audit')
|
|
35
|
-
.ifNotExists()
|
|
36
|
-
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
37
|
-
.addColumn('secret_key', 'varchar(255)', (col) => col.notNull())
|
|
38
|
-
.addColumn('action', 'varchar(20)', (col) => col.notNull())
|
|
39
|
-
.addColumn('performed_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
40
|
-
.execute();
|
|
41
|
-
}
|
|
23
|
+
await ensurePikkuSchema(this.db, secretSchema);
|
|
42
24
|
this.initialized = true;
|
|
43
25
|
}
|
|
44
26
|
async logAudit(secretKey, action) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { sql } from 'kysely';
|
|
2
1
|
import { parseJson } from './kysely-json.js';
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { sessionSchema } from './schema/session.schema.js';
|
|
3
4
|
export class KyselySessionStore {
|
|
4
5
|
db;
|
|
5
6
|
initialized = false;
|
|
@@ -7,17 +8,9 @@ export class KyselySessionStore {
|
|
|
7
8
|
this.db = db;
|
|
8
9
|
}
|
|
9
10
|
async init() {
|
|
10
|
-
if (this.initialized)
|
|
11
|
+
if (this.initialized)
|
|
11
12
|
return;
|
|
12
|
-
|
|
13
|
-
await this.db.schema
|
|
14
|
-
.createTable('pikku_user_sessions')
|
|
15
|
-
.ifNotExists()
|
|
16
|
-
.addColumn('pikku_user_id', 'text', (col) => col.primaryKey())
|
|
17
|
-
.addColumn('session', 'text', (col) => col.notNull())
|
|
18
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
19
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
20
|
-
.execute();
|
|
13
|
+
await ensurePikkuSchema(this.db, sessionSchema);
|
|
21
14
|
this.initialized = true;
|
|
22
15
|
}
|
|
23
16
|
async get(pikkuUserId) {
|
|
@@ -112,6 +112,8 @@ export interface AIRunTable {
|
|
|
112
112
|
errorMessage: string | null;
|
|
113
113
|
suspendReason: 'approval' | 'credential' | 'rpc-missing' | null;
|
|
114
114
|
missingRpcs: string | null;
|
|
115
|
+
/** JSON-serialized approvals the run is suspended on. */
|
|
116
|
+
pendingApprovals: string | null;
|
|
115
117
|
usageInputTokens: Generated<number>;
|
|
116
118
|
usageOutputTokens: Generated<number>;
|
|
117
119
|
usageModel: Generated<string>;
|
|
@@ -23,5 +23,4 @@ export declare class KyselyWebhookService extends QueueWebhookService {
|
|
|
23
23
|
}): Promise<WebhookDeliveryRecord[]>;
|
|
24
24
|
/** A single delivery with its full attempt history, or null if unknown. */
|
|
25
25
|
getDelivery(deliveryId: string): Promise<WebhookDeliveryWithAttempts | null>;
|
|
26
|
-
private createIndexSafe;
|
|
27
26
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { PIKKU_OUTGOING_WEBHOOK_QUEUE_NAME, QueueWebhookService, } from '@pikku/core/services';
|
|
2
|
-
import {
|
|
2
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
|
+
import { webhookSchema } from './schema/webhook.schema.js';
|
|
3
4
|
/**
|
|
4
5
|
* Durable {@link QueueWebhookService}: still delivers through the
|
|
5
6
|
* `pikku-outgoing-webhooks` queue, but records a `webhook_delivery` row per
|
|
@@ -15,43 +16,9 @@ export class KyselyWebhookService extends QueueWebhookService {
|
|
|
15
16
|
this.db = db;
|
|
16
17
|
}
|
|
17
18
|
async init() {
|
|
18
|
-
if (this.initialized)
|
|
19
|
+
if (this.initialized)
|
|
19
20
|
return;
|
|
20
|
-
|
|
21
|
-
await this.db.schema
|
|
22
|
-
.createTable('webhook_delivery')
|
|
23
|
-
.ifNotExists()
|
|
24
|
-
.addColumn('delivery_id', 'text', (col) => col.primaryKey())
|
|
25
|
-
.addColumn('organization_id', 'text')
|
|
26
|
-
.addColumn('url', 'text', (col) => col.notNull())
|
|
27
|
-
.addColumn('event', 'text')
|
|
28
|
-
.addColumn('status', 'text', (col) => col.defaultTo('pending').notNull())
|
|
29
|
-
.addColumn('attempts', 'integer', (col) => col.defaultTo(0).notNull())
|
|
30
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
31
|
-
.addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
32
|
-
.addColumn('delivered_at', 'timestamp')
|
|
33
|
-
.execute();
|
|
34
|
-
await this.db.schema
|
|
35
|
-
.createTable('webhook_delivery_attempt')
|
|
36
|
-
.ifNotExists()
|
|
37
|
-
.addColumn('attempt_id', 'text', (col) => col.primaryKey())
|
|
38
|
-
.addColumn('delivery_id', 'text', (col) => col.notNull().references('webhook_delivery.delivery_id').onDelete('cascade'))
|
|
39
|
-
.addColumn('attempt_number', 'integer', (col) => col.notNull())
|
|
40
|
-
.addColumn('status_code', 'integer')
|
|
41
|
-
.addColumn('response_body', 'text')
|
|
42
|
-
.addColumn('error', 'text')
|
|
43
|
-
.addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
44
|
-
.execute();
|
|
45
|
-
await this.createIndexSafe(this.db.schema
|
|
46
|
-
.createIndex('idx_webhook_delivery_org')
|
|
47
|
-
.ifNotExists()
|
|
48
|
-
.on('webhook_delivery')
|
|
49
|
-
.column('organization_id'));
|
|
50
|
-
await this.createIndexSafe(this.db.schema
|
|
51
|
-
.createIndex('idx_webhook_delivery_attempt_delivery')
|
|
52
|
-
.ifNotExists()
|
|
53
|
-
.on('webhook_delivery_attempt')
|
|
54
|
-
.column('delivery_id'));
|
|
21
|
+
await ensurePikkuSchema(this.db, webhookSchema);
|
|
55
22
|
this.initialized = true;
|
|
56
23
|
}
|
|
57
24
|
async send(input) {
|
|
@@ -133,18 +100,4 @@ export class KyselyWebhookService extends QueueWebhookService {
|
|
|
133
100
|
.execute();
|
|
134
101
|
return { delivery, attempts };
|
|
135
102
|
}
|
|
136
|
-
async createIndexSafe(builder) {
|
|
137
|
-
try {
|
|
138
|
-
await builder.execute();
|
|
139
|
-
}
|
|
140
|
-
catch (e) {
|
|
141
|
-
if (e?.code === 'ER_DUP_KEYNAME' || e?.errno === 1061)
|
|
142
|
-
return;
|
|
143
|
-
if (e?.code === '42P07')
|
|
144
|
-
return;
|
|
145
|
-
if (e?.message?.includes('already exists'))
|
|
146
|
-
return;
|
|
147
|
-
throw e;
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
103
|
}
|
|
@@ -19,9 +19,13 @@ export declare class KyselyWorkflowMirror implements WorkflowRunMirror {
|
|
|
19
19
|
private initialized;
|
|
20
20
|
constructor(db: Kysely<KyselyPikkuDB>);
|
|
21
21
|
/**
|
|
22
|
-
* Create the underlying tables if
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* Create the underlying tables if none of them exist yet.
|
|
23
|
+
*
|
|
24
|
+
* Safe to call from either the mirror or `KyselyWorkflowService.init()` —
|
|
25
|
+
* both apply the same `workflowSchema`, and `ensurePikkuSchema` is a no-op
|
|
26
|
+
* once the tables are there. A database holding only *part* of the schema
|
|
27
|
+
* throws rather than filling in the rest: something else owns those tables,
|
|
28
|
+
* and boot is not where that gets reconciled.
|
|
25
29
|
*/
|
|
26
30
|
init(): Promise<void>;
|
|
27
31
|
createRun(runId: string, workflowName: string, input: any, inline: boolean, graphHash: string, wire: WorkflowRunWire, options?: {
|