@pikku/kysely 0.13.3 → 0.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.js +1 -0
- package/dist/src/kysely-ai-run-state-service.js +3 -18
- package/dist/src/kysely-ai-storage-service.d.ts +0 -1
- package/dist/src/kysely-ai-storage-service.js +4 -101
- package/dist/src/kysely-channel-store.js +4 -24
- package/dist/src/kysely-credential-service.js +3 -24
- package/dist/src/kysely-deployment-service.d.ts +0 -1
- package/dist/src/kysely-deployment-service.js +4 -48
- package/dist/src/kysely-scope-service.js +4 -54
- package/dist/src/kysely-secret-service.js +3 -21
- package/dist/src/kysely-session-store.js +4 -11
- package/dist/src/kysely-tables.d.ts +2 -0
- package/dist/src/kysely-webhook-service.d.ts +0 -1
- package/dist/src/kysely-webhook-service.js +4 -51
- package/dist/src/kysely-workflow-mirror.d.ts +7 -3
- package/dist/src/kysely-workflow-mirror.js +10 -87
- package/dist/src/kysely-workflow-service.js +9 -93
- package/dist/src/schema/ai.schema.d.ts +12 -0
- package/dist/src/schema/ai.schema.js +89 -0
- package/dist/src/schema/channel.schema.d.ts +3 -0
- package/dist/src/schema/channel.schema.js +24 -0
- package/dist/src/schema/credential.schema.d.ts +11 -0
- package/dist/src/schema/credential.schema.js +33 -0
- package/dist/src/schema/deployment.schema.d.ts +3 -0
- package/dist/src/schema/deployment.schema.js +32 -0
- package/dist/src/schema/index.d.ts +99 -0
- package/dist/src/schema/index.js +274 -0
- package/dist/src/schema/pikku-schema.types.d.ts +75 -0
- package/dist/src/schema/pikku-schema.types.js +21 -0
- package/dist/src/schema/scope.schema.d.ts +18 -0
- package/dist/src/schema/scope.schema.js +61 -0
- package/dist/src/schema/secret.schema.d.ts +11 -0
- package/dist/src/schema/secret.schema.js +29 -0
- package/dist/src/schema/session.schema.d.ts +3 -0
- package/dist/src/schema/session.schema.js +13 -0
- package/dist/src/schema/webhook.schema.d.ts +3 -0
- package/dist/src/schema/webhook.schema.js +38 -0
- package/dist/src/schema/workflow.schema.d.ts +15 -0
- package/dist/src/schema/workflow.schema.js +86 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/index.ts +18 -0
- package/src/kysely-ai-run-state-service.ts +7 -28
- package/src/kysely-ai-storage-service.ts +4 -152
- package/src/kysely-channel-store.ts +4 -34
- package/src/kysely-credential-service.ts +3 -36
- package/src/kysely-deployment-service.ts +4 -63
- package/src/kysely-scope-service.ts +4 -81
- package/src/kysely-secret-service.ts +6 -33
- package/src/kysely-session-store.ts +4 -18
- package/src/kysely-tables.test.ts +110 -0
- package/src/kysely-tables.ts +2 -0
- package/src/kysely-webhook-service.ts +4 -67
- package/src/kysely-workflow-mirror.test.ts +7 -7
- package/src/kysely-workflow-mirror.ts +10 -109
- package/src/kysely-workflow-service.ts +12 -123
- package/src/schema/ai.schema.ts +140 -0
- package/src/schema/channel.schema.ts +37 -0
- package/src/schema/credential.schema.ts +46 -0
- package/src/schema/deployment.schema.ts +47 -0
- package/src/schema/index.ts +374 -0
- package/src/schema/pikku-schema.types.ts +94 -0
- package/src/schema/schema.test.ts +456 -0
- package/src/schema/scope.schema.ts +95 -0
- package/src/schema/secret.schema.ts +40 -0
- package/src/schema/session.schema.ts +20 -0
- package/src/schema/webhook.schema.ts +57 -0
- package/src/schema/workflow.schema.ts +113 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { sql } from 'kysely'
|
|
2
|
+
import type { PikkuSchema } from './pikku-schema.types.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Threads, messages, tool calls, working memory and agent runs.
|
|
6
|
+
*
|
|
7
|
+
* `aiRun` carries `pendingApprovals` even though the two services that used to
|
|
8
|
+
* create this table disagreed about it: `KyselyAIStorageService` omitted the
|
|
9
|
+
* column while `KyselyAIRunStateService` declared it and read it back through
|
|
10
|
+
* casts. Whichever service happened to run first decided the shape, and on a
|
|
11
|
+
* database where the storage service won, resolving an approval failed. One
|
|
12
|
+
* declaration ends that — the column is here because live code reads it.
|
|
13
|
+
*/
|
|
14
|
+
export const aiSchema: PikkuSchema = {
|
|
15
|
+
name: 'ai',
|
|
16
|
+
statements: [
|
|
17
|
+
(db) =>
|
|
18
|
+
db.schema
|
|
19
|
+
.createTable('aiThreads')
|
|
20
|
+
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
21
|
+
.addColumn('resourceId', 'varchar(255)', (col) => col.notNull())
|
|
22
|
+
.addColumn('title', 'text')
|
|
23
|
+
.addColumn('metadata', 'text')
|
|
24
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
25
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
26
|
+
)
|
|
27
|
+
.addColumn('updatedAt', 'timestamp', (col) =>
|
|
28
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
29
|
+
),
|
|
30
|
+
|
|
31
|
+
(db) =>
|
|
32
|
+
db.schema
|
|
33
|
+
.createIndex('idx_ai_threads_resource')
|
|
34
|
+
.on('aiThreads')
|
|
35
|
+
.column('resourceId'),
|
|
36
|
+
|
|
37
|
+
(db) =>
|
|
38
|
+
db.schema
|
|
39
|
+
.createTable('aiMessage')
|
|
40
|
+
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
41
|
+
.addColumn('threadId', 'varchar(36)', (col) =>
|
|
42
|
+
col.notNull().references('aiThreads.id').onDelete('cascade')
|
|
43
|
+
)
|
|
44
|
+
.addColumn('role', 'varchar(50)', (col) => col.notNull())
|
|
45
|
+
.addColumn('content', 'text')
|
|
46
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
47
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
48
|
+
),
|
|
49
|
+
|
|
50
|
+
(db) =>
|
|
51
|
+
db.schema
|
|
52
|
+
.createIndex('idx_ai_message_thread')
|
|
53
|
+
.on('aiMessage')
|
|
54
|
+
.columns(['threadId', 'createdAt']),
|
|
55
|
+
|
|
56
|
+
(db) =>
|
|
57
|
+
db.schema
|
|
58
|
+
.createTable('aiToolCall')
|
|
59
|
+
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
60
|
+
.addColumn('threadId', 'varchar(36)', (col) =>
|
|
61
|
+
col.notNull().references('aiThreads.id').onDelete('cascade')
|
|
62
|
+
)
|
|
63
|
+
.addColumn('messageId', 'varchar(36)', (col) =>
|
|
64
|
+
col.notNull().references('aiMessage.id').onDelete('cascade')
|
|
65
|
+
)
|
|
66
|
+
.addColumn('runId', 'varchar(36)')
|
|
67
|
+
.addColumn('toolName', 'varchar(255)', (col) => col.notNull())
|
|
68
|
+
.addColumn('args', 'text', (col) => col.notNull())
|
|
69
|
+
.addColumn('result', 'text')
|
|
70
|
+
.addColumn('approvalStatus', 'varchar(50)')
|
|
71
|
+
.addColumn('approvalType', 'varchar(50)')
|
|
72
|
+
.addColumn('agentRunId', 'varchar(36)')
|
|
73
|
+
.addColumn('displayToolName', 'varchar(255)')
|
|
74
|
+
.addColumn('displayArgs', 'text')
|
|
75
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
76
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
77
|
+
),
|
|
78
|
+
|
|
79
|
+
(db) =>
|
|
80
|
+
db.schema
|
|
81
|
+
.createIndex('idx_ai_tool_call_thread')
|
|
82
|
+
.on('aiToolCall')
|
|
83
|
+
.column('threadId'),
|
|
84
|
+
|
|
85
|
+
(db) =>
|
|
86
|
+
db.schema
|
|
87
|
+
.createIndex('idx_ai_tool_call_message')
|
|
88
|
+
.on('aiToolCall')
|
|
89
|
+
.column('messageId'),
|
|
90
|
+
|
|
91
|
+
(db) =>
|
|
92
|
+
db.schema
|
|
93
|
+
.createTable('aiWorkingMemory')
|
|
94
|
+
.addColumn('id', 'varchar(255)', (col) => col.notNull())
|
|
95
|
+
.addColumn('scope', 'varchar(50)', (col) => col.notNull())
|
|
96
|
+
.addColumn('data', 'text', (col) => col.notNull())
|
|
97
|
+
.addColumn('updatedAt', 'timestamp', (col) =>
|
|
98
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
99
|
+
)
|
|
100
|
+
.addPrimaryKeyConstraint('ai_working_memory_pk', ['id', 'scope']),
|
|
101
|
+
|
|
102
|
+
(db) =>
|
|
103
|
+
db.schema
|
|
104
|
+
.createTable('aiRun')
|
|
105
|
+
.addColumn('runId', 'varchar(36)', (col) => col.primaryKey())
|
|
106
|
+
.addColumn('agentName', 'varchar(255)', (col) => col.notNull())
|
|
107
|
+
.addColumn('threadId', 'varchar(36)', (col) =>
|
|
108
|
+
col.notNull().references('aiThreads.id').onDelete('cascade')
|
|
109
|
+
)
|
|
110
|
+
.addColumn('resourceId', 'varchar(255)', (col) => col.notNull())
|
|
111
|
+
.addColumn('status', 'varchar(50)', (col) =>
|
|
112
|
+
col.notNull().defaultTo('running')
|
|
113
|
+
)
|
|
114
|
+
.addColumn('errorMessage', 'text')
|
|
115
|
+
.addColumn('suspendReason', 'text')
|
|
116
|
+
.addColumn('missingRpcs', 'text')
|
|
117
|
+
.addColumn('pendingApprovals', 'text')
|
|
118
|
+
.addColumn('usageInputTokens', 'integer', (col) =>
|
|
119
|
+
col.notNull().defaultTo(0)
|
|
120
|
+
)
|
|
121
|
+
.addColumn('usageOutputTokens', 'integer', (col) =>
|
|
122
|
+
col.notNull().defaultTo(0)
|
|
123
|
+
)
|
|
124
|
+
.addColumn('usageModel', 'varchar(255)', (col) =>
|
|
125
|
+
col.notNull().defaultTo('')
|
|
126
|
+
)
|
|
127
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
128
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
129
|
+
)
|
|
130
|
+
.addColumn('updatedAt', 'timestamp', (col) =>
|
|
131
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
132
|
+
),
|
|
133
|
+
|
|
134
|
+
(db) =>
|
|
135
|
+
db.schema
|
|
136
|
+
.createIndex('idx_ai_run_thread')
|
|
137
|
+
.on('aiRun')
|
|
138
|
+
.columns(['threadId', 'createdAt']),
|
|
139
|
+
],
|
|
140
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { sql } from 'kysely'
|
|
2
|
+
import type { PikkuSchema } from './pikku-schema.types.js'
|
|
3
|
+
|
|
4
|
+
/** Channels and their topic subscriptions, also read by the event-hub store. */
|
|
5
|
+
export const channelSchema: PikkuSchema = {
|
|
6
|
+
name: 'channel',
|
|
7
|
+
statements: [
|
|
8
|
+
(db) =>
|
|
9
|
+
db.schema
|
|
10
|
+
.createTable('channels')
|
|
11
|
+
.addColumn('channelId', 'text', (col) => col.primaryKey())
|
|
12
|
+
.addColumn('channelName', 'text', (col) => col.notNull())
|
|
13
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
14
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
15
|
+
)
|
|
16
|
+
.addColumn('openingData', 'text', (col) =>
|
|
17
|
+
col.notNull().defaultTo('{}')
|
|
18
|
+
)
|
|
19
|
+
.addColumn('pikkuUserId', 'text')
|
|
20
|
+
.addColumn('state', 'text')
|
|
21
|
+
.addColumn('lastWire', 'timestamp', (col) =>
|
|
22
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
23
|
+
),
|
|
24
|
+
|
|
25
|
+
(db) =>
|
|
26
|
+
db.schema
|
|
27
|
+
.createTable('channelSubscriptions')
|
|
28
|
+
.addColumn('channelId', 'text', (col) =>
|
|
29
|
+
col.notNull().references('channels.channelId').onDelete('cascade')
|
|
30
|
+
)
|
|
31
|
+
.addColumn('topic', 'text', (col) => col.notNull())
|
|
32
|
+
.addPrimaryKeyConstraint('channel_subscriptions_pk', [
|
|
33
|
+
'channelId',
|
|
34
|
+
'topic',
|
|
35
|
+
]),
|
|
36
|
+
],
|
|
37
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { sql } from 'kysely'
|
|
2
|
+
import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Encrypted per-user credentials and their audit trail.
|
|
6
|
+
*
|
|
7
|
+
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
8
|
+
* rows with a null user must still collide on name — which the schema builder
|
|
9
|
+
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
10
|
+
* which is why the table is named explicitly rather than left to the search
|
|
11
|
+
* path.
|
|
12
|
+
*/
|
|
13
|
+
export const credentialSchema: PikkuSchema = {
|
|
14
|
+
name: 'credential',
|
|
15
|
+
statements: [
|
|
16
|
+
(db) =>
|
|
17
|
+
db.schema
|
|
18
|
+
.createTable('credentials')
|
|
19
|
+
.addColumn('name', 'varchar(255)', (col) => col.notNull())
|
|
20
|
+
.addColumn('userId', 'varchar(255)')
|
|
21
|
+
.addColumn('ciphertext', 'text', (col) => col.notNull())
|
|
22
|
+
.addColumn('wrappedDek', 'text', (col) => col.notNull())
|
|
23
|
+
.addColumn('keyVersion', 'integer', (col) => col.notNull())
|
|
24
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
25
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
26
|
+
)
|
|
27
|
+
.addColumn('updatedAt', 'timestamp', (col) =>
|
|
28
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
29
|
+
),
|
|
30
|
+
|
|
31
|
+
rawStatement(
|
|
32
|
+
sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`
|
|
33
|
+
),
|
|
34
|
+
|
|
35
|
+
(db) =>
|
|
36
|
+
db.schema
|
|
37
|
+
.createTable('credentialsAudit')
|
|
38
|
+
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
39
|
+
.addColumn('credentialName', 'varchar(255)', (col) => col.notNull())
|
|
40
|
+
.addColumn('userId', 'varchar(255)')
|
|
41
|
+
.addColumn('action', 'varchar(20)', (col) => col.notNull())
|
|
42
|
+
.addColumn('performedAt', 'timestamp', (col) =>
|
|
43
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
44
|
+
),
|
|
45
|
+
],
|
|
46
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { sql } from 'kysely'
|
|
2
|
+
import type { PikkuSchema } from './pikku-schema.types.js'
|
|
3
|
+
|
|
4
|
+
/** Registered deployments and the functions each one serves. */
|
|
5
|
+
export const deploymentSchema: PikkuSchema = {
|
|
6
|
+
name: 'deployment',
|
|
7
|
+
statements: [
|
|
8
|
+
(db) =>
|
|
9
|
+
db.schema
|
|
10
|
+
.createTable('pikkuDeployments')
|
|
11
|
+
.addColumn('deploymentId', 'text', (col) => col.primaryKey())
|
|
12
|
+
.addColumn('endpoint', 'text', (col) => col.notNull())
|
|
13
|
+
.addColumn('lastHeartbeat', 'timestamp', (col) =>
|
|
14
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
15
|
+
)
|
|
16
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
17
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
18
|
+
),
|
|
19
|
+
|
|
20
|
+
(db) =>
|
|
21
|
+
db.schema
|
|
22
|
+
.createTable('pikkuDeploymentFunctions')
|
|
23
|
+
.addColumn('deploymentId', 'text', (col) =>
|
|
24
|
+
col
|
|
25
|
+
.notNull()
|
|
26
|
+
.references('pikkuDeployments.deploymentId')
|
|
27
|
+
.onDelete('cascade')
|
|
28
|
+
)
|
|
29
|
+
.addColumn('functionName', 'text', (col) => col.notNull())
|
|
30
|
+
.addPrimaryKeyConstraint('pikku_deployment_functions_pk', [
|
|
31
|
+
'deploymentId',
|
|
32
|
+
'functionName',
|
|
33
|
+
]),
|
|
34
|
+
|
|
35
|
+
(db) =>
|
|
36
|
+
db.schema
|
|
37
|
+
.createIndex('idx_pikku_deployments_heartbeat')
|
|
38
|
+
.on('pikkuDeployments')
|
|
39
|
+
.column('lastHeartbeat'),
|
|
40
|
+
|
|
41
|
+
(db) =>
|
|
42
|
+
db.schema
|
|
43
|
+
.createIndex('idx_pikku_deployment_functions_name')
|
|
44
|
+
.on('pikkuDeploymentFunctions')
|
|
45
|
+
.column('functionName'),
|
|
46
|
+
],
|
|
47
|
+
}
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
import { CamelCasePlugin, type Kysely } from 'kysely'
|
|
2
|
+
import type {
|
|
3
|
+
PikkuSchema,
|
|
4
|
+
RequiredTypes,
|
|
5
|
+
SchemaRequirement,
|
|
6
|
+
} from './pikku-schema.types.js'
|
|
7
|
+
import { aiSchema } from './ai.schema.js'
|
|
8
|
+
import { channelSchema } from './channel.schema.js'
|
|
9
|
+
import { credentialSchema } from './credential.schema.js'
|
|
10
|
+
import { deploymentSchema } from './deployment.schema.js'
|
|
11
|
+
import { scopeSchema } from './scope.schema.js'
|
|
12
|
+
import { secretSchema } from './secret.schema.js'
|
|
13
|
+
import { sessionSchema } from './session.schema.js'
|
|
14
|
+
import { webhookSchema } from './webhook.schema.js'
|
|
15
|
+
import { workflowSchema } from './workflow.schema.js'
|
|
16
|
+
|
|
17
|
+
export type {
|
|
18
|
+
PikkuSchema,
|
|
19
|
+
RequiredTypes,
|
|
20
|
+
SchemaRequirement,
|
|
21
|
+
SchemaStatement,
|
|
22
|
+
SchemaStatementFactory,
|
|
23
|
+
} from './pikku-schema.types.js'
|
|
24
|
+
export { rawStatement, requiredType } from './pikku-schema.types.js'
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Every table the pikku runtime needs, in dependency order.
|
|
28
|
+
*
|
|
29
|
+
* Order is load-bearing, not cosmetic: `scope` has foreign keys onto Better
|
|
30
|
+
* Auth's `user`, and the tables within each schema reference each other. Sort
|
|
31
|
+
* this list alphabetically and it stops applying.
|
|
32
|
+
*/
|
|
33
|
+
export const pikkuSchemas: PikkuSchema[] = [
|
|
34
|
+
channelSchema,
|
|
35
|
+
sessionSchema,
|
|
36
|
+
secretSchema,
|
|
37
|
+
credentialSchema,
|
|
38
|
+
deploymentSchema,
|
|
39
|
+
webhookSchema,
|
|
40
|
+
workflowSchema,
|
|
41
|
+
aiSchema,
|
|
42
|
+
scopeSchema,
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Bind a declaration to a database.
|
|
47
|
+
*
|
|
48
|
+
* The plugin is applied here rather than trusted to the caller. Declarations
|
|
49
|
+
* name tables and columns the way the rest of the codebase queries them
|
|
50
|
+
* (`aiThreads.resourceId`), and it is `CamelCasePlugin` that turns those into
|
|
51
|
+
* the physical `ai_threads.resource_id`. Without it the same declaration
|
|
52
|
+
* silently creates quoted camelCase tables that nothing can read.
|
|
53
|
+
*/
|
|
54
|
+
const bind = (db: Kysely<any>) => db.withPlugin(new CamelCasePlugin())
|
|
55
|
+
|
|
56
|
+
export interface UnmetRequirement {
|
|
57
|
+
schema: PikkuSchema
|
|
58
|
+
requirement: SchemaRequirement
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Look up every prerequisite in `db`, reporting the gaps and the types found.
|
|
63
|
+
*
|
|
64
|
+
* Both halves come from one introspection because they are one question. The
|
|
65
|
+
* gaps say whether a schema can be applied at all; the types say how — a
|
|
66
|
+
* foreign key onto a `uuid` primary key has to be declared `uuid`, and guessing
|
|
67
|
+
* `text` produces DDL the database refuses.
|
|
68
|
+
*
|
|
69
|
+
* Introspection returns physical names, so `requires` is written in physical
|
|
70
|
+
* names too — this is the one place in the declaration that is not camelCase.
|
|
71
|
+
*/
|
|
72
|
+
export const resolveRequirements = async (
|
|
73
|
+
db: Kysely<any>,
|
|
74
|
+
schemas: PikkuSchema[] = pikkuSchemas
|
|
75
|
+
): Promise<{ types: RequiredTypes; unmet: UnmetRequirement[] }> => {
|
|
76
|
+
const tables = new Map(
|
|
77
|
+
(await bind(db).introspection.getTables()).map((table) => [
|
|
78
|
+
table.name,
|
|
79
|
+
table,
|
|
80
|
+
])
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
const types: RequiredTypes = {}
|
|
84
|
+
const unmet: UnmetRequirement[] = []
|
|
85
|
+
for (const schema of schemas) {
|
|
86
|
+
for (const requirement of schema.requires ?? []) {
|
|
87
|
+
const column = tables
|
|
88
|
+
.get(requirement.table)
|
|
89
|
+
?.columns.find((c) => c.name === requirement.column)
|
|
90
|
+
if (!column) {
|
|
91
|
+
unmet.push({ schema, requirement })
|
|
92
|
+
} else {
|
|
93
|
+
types[`${requirement.table}.${requirement.column}`] = column.dataType
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return { types, unmet }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The prerequisites `db` does not satisfy, as a list rather than a throw.
|
|
102
|
+
*
|
|
103
|
+
* Separate from `applyPikkuSchemas` because the two callers want opposite
|
|
104
|
+
* things from the same answer: a service booting cannot proceed without its
|
|
105
|
+
* tables, while a tool asking what a project's schema *would* be needs to
|
|
106
|
+
* describe the gap rather than die on it.
|
|
107
|
+
*/
|
|
108
|
+
export const unsatisfiedRequirements = async (
|
|
109
|
+
db: Kysely<any>,
|
|
110
|
+
schemas: PikkuSchema[] = pikkuSchemas
|
|
111
|
+
): Promise<UnmetRequirement[]> => (await resolveRequirements(db, schemas)).unmet
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Create the tables, in order, on a database that has whatever they require.
|
|
115
|
+
*
|
|
116
|
+
* Prerequisites are checked before anything is created, so a project missing
|
|
117
|
+
* one is told which schema wanted what — rather than being handed a foreign key
|
|
118
|
+
* error, or a half-applied database to clean up.
|
|
119
|
+
*
|
|
120
|
+
* The statements then run in one transaction, so the same promise holds for a
|
|
121
|
+
* failure the check cannot foresee: a statement that dies mid-list takes the
|
|
122
|
+
* ones before it with it. On an engine without transactional DDL the rollback
|
|
123
|
+
* is the engine's to give, not ours.
|
|
124
|
+
*/
|
|
125
|
+
export const applyPikkuSchemas = async (
|
|
126
|
+
db: Kysely<any>,
|
|
127
|
+
schemas: PikkuSchema[] = pikkuSchemas
|
|
128
|
+
): Promise<void> => {
|
|
129
|
+
const { types, unmet } = await resolveRequirements(db, schemas)
|
|
130
|
+
if (unmet.length > 0) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
unmet
|
|
133
|
+
.map(
|
|
134
|
+
({ schema, requirement }) =>
|
|
135
|
+
`The '${schema.name}' schema requires '${requirement.table}.${requirement.column}', which nothing has created. ` +
|
|
136
|
+
`${requirement.owner} owns it — apply its schema first.`
|
|
137
|
+
)
|
|
138
|
+
.join('\n')
|
|
139
|
+
)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
await db.transaction().execute(async (trx) => {
|
|
143
|
+
const bound = bind(trx)
|
|
144
|
+
for (const schema of schemas) {
|
|
145
|
+
for (const statement of schema.statements) {
|
|
146
|
+
const query = statement(bound, types)
|
|
147
|
+
try {
|
|
148
|
+
await query.execute()
|
|
149
|
+
} catch (error) {
|
|
150
|
+
throw explain(error, query.compile().sql, schema)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
})
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** `references "app"."workflow_runs"` — a foreign key onto a qualified table. */
|
|
158
|
+
const QUALIFIED_REFERENCE = /references "[^"]+"\."[^"]+"/i
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* sqlite's own words for "you put a dot where a table name goes".
|
|
162
|
+
*
|
|
163
|
+
* Matched rather than a driver's `code`, because every sqlite binding
|
|
164
|
+
* (better-sqlite3, node:sqlite, bun:sqlite, libsql) passes the engine's message
|
|
165
|
+
* through verbatim while disagreeing about how to label it. Postgres phrases the
|
|
166
|
+
* same class of error the other way round — `syntax error at or near "."` — so
|
|
167
|
+
* this cannot fire on it.
|
|
168
|
+
*/
|
|
169
|
+
const SQLITE_REJECTED_QUALIFIER = /near "\.":\s*syntax error/i
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Say what a schema-bound connection did to the DDL, when that is what failed.
|
|
173
|
+
*
|
|
174
|
+
* `withSchema(...)` qualifies foreign key targets along with everything else.
|
|
175
|
+
* Postgres needs that — an unqualified reference resolves against `search_path`,
|
|
176
|
+
* which a schema-bound connection does not control — and sqlite refuses it: a
|
|
177
|
+
* `REFERENCES` clause there takes a bare table name, because a foreign key can
|
|
178
|
+
* only point inside the same database, so there is no qualifier to give. One
|
|
179
|
+
* declaration cannot spell both, and what the engine says about it is
|
|
180
|
+
* `near ".": syntax error`, which names neither the schema nor the cause.
|
|
181
|
+
*
|
|
182
|
+
* Gated on sqlite's exact phrasing as well as the SQL. Postgres compiles the
|
|
183
|
+
* same qualified reference legitimately, and can raise a syntax error of its own
|
|
184
|
+
* on this DDL — a column type resolved from `requires` goes in verbatim — so a
|
|
185
|
+
* looser `/syntax error/` would answer for postgres in sqlite's name.
|
|
186
|
+
*/
|
|
187
|
+
const explain = (error: unknown, sql: string, schema: PikkuSchema): unknown => {
|
|
188
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
189
|
+
if (
|
|
190
|
+
!QUALIFIED_REFERENCE.test(sql) ||
|
|
191
|
+
!SQLITE_REJECTED_QUALIFIER.test(message)
|
|
192
|
+
) {
|
|
193
|
+
return error
|
|
194
|
+
}
|
|
195
|
+
return new Error(
|
|
196
|
+
`The '${schema.name}' schema has a foreign key, and this connection is bound to a schema, ` +
|
|
197
|
+
`so it compiled to a qualified 'references "schema"."table"' that the database rejected. ` +
|
|
198
|
+
'sqlite is the engine that does this: its REFERENCES clause takes a bare table name. ' +
|
|
199
|
+
'Use a connection without withSchema(...) there — sqlite has one schema to be in — ' +
|
|
200
|
+
'or write the schema down as a migration with `pikku db generate` and apply it yourself. ' +
|
|
201
|
+
`The database said: ${message}`,
|
|
202
|
+
{ cause: error }
|
|
203
|
+
)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** A table a schema creates, as the DDL addresses it. */
|
|
207
|
+
export interface DeclaredTable {
|
|
208
|
+
/** Present when the connection is `withSchema(...)`-bound. */
|
|
209
|
+
schema?: string
|
|
210
|
+
name: string
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** How a table reads in an error message: qualified only when it is. */
|
|
214
|
+
const displayName = (table: DeclaredTable): string =>
|
|
215
|
+
table.schema ? `${table.schema}.${table.name}` : table.name
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* `create table "workflow_runs"` and `create table "app"."workflow_runs"`.
|
|
219
|
+
*
|
|
220
|
+
* The schema half is optional and the table is always the LAST quoted
|
|
221
|
+
* identifier — matching the first one instead reads a `withSchema('app')`
|
|
222
|
+
* connection's DDL as a table called `app`, which is no table at all.
|
|
223
|
+
*/
|
|
224
|
+
const CREATE_TABLE =
|
|
225
|
+
/^\s*create table (?:if not exists\s+)?(?:"([^"]+)"\.)?"([^"]+)"/i
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* The tables a schema creates, read back out of its own compiled SQL.
|
|
229
|
+
*
|
|
230
|
+
* Derived rather than declared alongside the statements, because a hand-kept
|
|
231
|
+
* list of names is a second source of truth: rename a table in a statement and
|
|
232
|
+
* the list keeps answering for the old one.
|
|
233
|
+
*
|
|
234
|
+
* Compiled against `db` rather than in the abstract, so the answer carries the
|
|
235
|
+
* schema the DDL will actually target — the same binding the lookup has to use.
|
|
236
|
+
*/
|
|
237
|
+
export const declaredTables = (
|
|
238
|
+
schema: PikkuSchema,
|
|
239
|
+
db: Kysely<any>
|
|
240
|
+
): DeclaredTable[] => {
|
|
241
|
+
const bound = bind(db)
|
|
242
|
+
const tables: DeclaredTable[] = []
|
|
243
|
+
for (const statement of schema.statements) {
|
|
244
|
+
const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql)
|
|
245
|
+
if (match) tables.push({ schema: match[1], name: match[2]! })
|
|
246
|
+
}
|
|
247
|
+
return tables
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** What `ensurePikkuSchema` found when it looked. */
|
|
251
|
+
export type EnsureOutcome = 'present' | 'created'
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* What `db` already has, indexed both ways.
|
|
255
|
+
*
|
|
256
|
+
* Introspection reports every schema it can see, so the bare name alone cannot
|
|
257
|
+
* answer the question: a `workflow_runs` in some other schema is not the one a
|
|
258
|
+
* `withSchema('app')` connection reads or writes. Qualified declarations are
|
|
259
|
+
* matched on the pair. Unqualified DDL resolves against the connection's
|
|
260
|
+
* search_path, which is not knowable from here, so it falls back to the name.
|
|
261
|
+
*/
|
|
262
|
+
interface ExistingTables {
|
|
263
|
+
qualified: Set<string>
|
|
264
|
+
bare: Set<string>
|
|
265
|
+
/** Whether this engine reports schemas at all. */
|
|
266
|
+
schemaAware: boolean
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const tablesOf = async (db: Kysely<any>): Promise<ExistingTables> => {
|
|
270
|
+
const tables = await bind(db).introspection.getTables()
|
|
271
|
+
return {
|
|
272
|
+
qualified: new Set(
|
|
273
|
+
tables
|
|
274
|
+
.filter((table) => table.schema)
|
|
275
|
+
.map((table) => `${table.schema}.${table.name}`)
|
|
276
|
+
),
|
|
277
|
+
bare: new Set(tables.map((table) => table.name)),
|
|
278
|
+
schemaAware: tables.some((table) => table.schema !== undefined),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Qualified declarations match on the pair — but only where that pair means
|
|
284
|
+
* something.
|
|
285
|
+
*
|
|
286
|
+
* `TableMetadata.schema` is optional, and sqlite and mysql leave it unset:
|
|
287
|
+
* sqlite has one schema to be in, and mysql calls the thing a database. A
|
|
288
|
+
* `withSchema('main')` connection there declares `main.workflow_runs` while
|
|
289
|
+
* introspection can only offer `workflow_runs`, and holding out for the pair
|
|
290
|
+
* would read every existing table as missing — the same false negative this
|
|
291
|
+
* whole function exists to fix, arrived at from the other side. So when the
|
|
292
|
+
* engine reports no schemas, the bare name is the only name there is.
|
|
293
|
+
*/
|
|
294
|
+
const isPresent = (existing: ExistingTables, table: DeclaredTable): boolean =>
|
|
295
|
+
table.schema && existing.schemaAware
|
|
296
|
+
? existing.qualified.has(`${table.schema}.${table.name}`)
|
|
297
|
+
: existing.bare.has(table.name)
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Make sure one schema's tables are there, creating them only if none are.
|
|
301
|
+
*
|
|
302
|
+
* What a service calls at boot. It looks before it creates, which is the whole
|
|
303
|
+
* difference from the `.ifNotExists()` DDL this replaces: that spelling turned
|
|
304
|
+
* every failure into a silent no-op, and hid a foreign key onto `user.id` that
|
|
305
|
+
* postgres had been rejecting since the day it was written.
|
|
306
|
+
*
|
|
307
|
+
* A partially-present schema is refused rather than completed. Half a schema
|
|
308
|
+
* means something else already applied part of it — a migration, an older
|
|
309
|
+
* version, a hand-run script — and creating the remainder at boot would leave
|
|
310
|
+
* two authorities over one set of tables, which is the condition all of this
|
|
311
|
+
* exists to end.
|
|
312
|
+
*
|
|
313
|
+
* Creating at boot is the fallback, not the intent. `pikku db generate` writes
|
|
314
|
+
* the declaration down as a migration; a project that has done so takes the
|
|
315
|
+
* `present` path and this never issues DDL at all.
|
|
316
|
+
*/
|
|
317
|
+
export const ensurePikkuSchema = async (
|
|
318
|
+
db: Kysely<any>,
|
|
319
|
+
schema: PikkuSchema
|
|
320
|
+
): Promise<EnsureOutcome> => {
|
|
321
|
+
const existing = await tablesOf(db)
|
|
322
|
+
const declared = declaredTables(schema, db)
|
|
323
|
+
const missing = declared.filter((table) => !isPresent(existing, table))
|
|
324
|
+
|
|
325
|
+
if (missing.length === 0) return 'present'
|
|
326
|
+
|
|
327
|
+
if (missing.length < declared.length) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
`The '${schema.name}' schema is half applied: ${missing.map(displayName).join(', ')} missing, ` +
|
|
330
|
+
`${declared
|
|
331
|
+
.filter((table) => isPresent(existing, table))
|
|
332
|
+
.map(displayName)
|
|
333
|
+
.join(', ')} already there. ` +
|
|
334
|
+
'Something else owns part of it. Reconcile it with a migration — ' +
|
|
335
|
+
'`pikku db generate` writes the declaration down — rather than creating the rest at boot.'
|
|
336
|
+
)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
try {
|
|
340
|
+
await applyPikkuSchemas(db, [schema])
|
|
341
|
+
return 'created'
|
|
342
|
+
} catch (error) {
|
|
343
|
+
// Two instances booting cold against one database can both see it empty
|
|
344
|
+
// before either has finished creating. If the other one got there, say so;
|
|
345
|
+
// anything else is a real failure and still throws. This is a courtesy, not
|
|
346
|
+
// a guarantee — a boot race is only fully answered by having run the
|
|
347
|
+
// migration, which is what `pikku db generate` is for.
|
|
348
|
+
const after = await tablesOf(db)
|
|
349
|
+
if (declared.every((table) => isPresent(after, table))) return 'present'
|
|
350
|
+
throw error
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Render the declaration as SQL for the dialect `db` was built with.
|
|
356
|
+
*
|
|
357
|
+
* `types` comes from `resolveRequirements` against the database the migration
|
|
358
|
+
* is being written for. Omit it and foreign keys onto another source's columns
|
|
359
|
+
* compile as `text`, which is a shape, not a migration.
|
|
360
|
+
*/
|
|
361
|
+
export const compilePikkuSchemas = (
|
|
362
|
+
db: Kysely<any>,
|
|
363
|
+
schemas: PikkuSchema[] = pikkuSchemas,
|
|
364
|
+
types: RequiredTypes = {}
|
|
365
|
+
): string => {
|
|
366
|
+
const bound = bind(db)
|
|
367
|
+
return schemas
|
|
368
|
+
.flatMap((schema) =>
|
|
369
|
+
schema.statements.map(
|
|
370
|
+
(statement) => `${statement(bound, types).compile().sql};`
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
.join('\n\n')
|
|
374
|
+
}
|