@sixb/sqlite 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-storage/access-tokens.d.ts +0 -1
- package/dist/auth-storage/access-tokens.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +7 -4
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +8 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/file-upload-session-storage.d.ts +6 -0
- package/dist/file-upload-session-storage.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +441 -123
- package/dist/index.js.map +18 -17
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +0 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -1
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/workflow-intervention-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/auth-storage/access-tokens.ts +11 -18
- package/src/auth-storage/auth-storage.ts +49 -86
- package/src/auth-storage/device-authorizations.ts +3 -5
- package/src/auth-storage/rows.ts +18 -8
- package/src/auth-storage/sessions.ts +89 -4
- package/src/auth-storage/shared.ts +9 -3
- package/src/file-upload-session-storage.ts +136 -0
- package/src/index.ts +5 -0
- package/src/migrations/046-workflow-intervention-principals.sql +15 -0
- package/src/migrations/047-ontology-commit-attribution.sql +67 -0
- package/src/migrations/048-file-upload-sessions.sql +35 -0
- package/src/migrations/049-native-sessions.sql +12 -0
- package/src/migrations/050-directory-group-memberships.sql +21 -0
- package/src/migrations.ts +16 -0
- package/src/objects/query-compiler.ts +59 -4
- package/src/objects/reader.ts +4 -0
- package/src/objects/storage.ts +1 -0
- package/src/ontology-storage/materializations.ts +16 -59
- package/src/ontology-storage/shared.ts +7 -3
- package/src/workflow-intervention-storage.ts +9 -9
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import {
|
|
3
|
+
DurableFileUploadSessions,
|
|
4
|
+
type FileUploadSessionPersistence,
|
|
5
|
+
type FileUploadSessionPersistenceBackend,
|
|
6
|
+
fileUploadSessionReapAt,
|
|
7
|
+
parseFileUploadSessionRow,
|
|
8
|
+
} from "@sixb/core/internal/file-upload-session-storage-provider"
|
|
9
|
+
import type { FileUploadSession, ListAbandonedFileUploadSessionsInput } from "@sixb/core/storage"
|
|
10
|
+
import { isUniqueConstraintError } from "./storage-errors"
|
|
11
|
+
import { runImmediateTransactionAsync, type SqliteStoreConnection } from "./transactions"
|
|
12
|
+
|
|
13
|
+
export class SqliteFileUploadSessionStorage extends DurableFileUploadSessions {
|
|
14
|
+
constructor(connection: SqliteStoreConnection) {
|
|
15
|
+
super(new SqliteFileUploadSessionBackend(connection.db))
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
class SqliteFileUploadSessionBackend implements FileUploadSessionPersistenceBackend {
|
|
20
|
+
private readonly persistence: FileUploadSessionPersistence
|
|
21
|
+
|
|
22
|
+
constructor(private readonly db: Database) {
|
|
23
|
+
this.persistence = new SqliteFileUploadSessionPersistence(db)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
read<T>(run: (persistence: FileUploadSessionPersistence) => Promise<T>): Promise<T> {
|
|
27
|
+
return run(this.persistence)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
transaction<T>(run: (persistence: FileUploadSessionPersistence) => Promise<T>): Promise<T> {
|
|
31
|
+
return runImmediateTransactionAsync(this.db, () => run(this.persistence))
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
class SqliteFileUploadSessionPersistence implements FileUploadSessionPersistence {
|
|
36
|
+
constructor(private readonly db: Database) {}
|
|
37
|
+
|
|
38
|
+
async now(): Promise<Date> {
|
|
39
|
+
const row = this.db.query("SELECT strftime('%Y-%m-%dT%H:%M:%fZ', 'now') AS now").get() as {
|
|
40
|
+
readonly now: string
|
|
41
|
+
}
|
|
42
|
+
return new Date(row.now)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async insert(session: FileUploadSession): Promise<boolean> {
|
|
46
|
+
try {
|
|
47
|
+
this.db
|
|
48
|
+
.query(
|
|
49
|
+
`INSERT INTO file_upload_sessions (
|
|
50
|
+
id, project_id, owner_principal_type, owner_principal_id, strategy, status, file_name,
|
|
51
|
+
media_type, logical_path, expected_size_bytes, expected_digest, provider_upload, signed_parts, file_ref,
|
|
52
|
+
created_at, expires_at, completed_at, aborted_at, reap_at
|
|
53
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
54
|
+
)
|
|
55
|
+
.run(
|
|
56
|
+
session.id,
|
|
57
|
+
session.projectId,
|
|
58
|
+
session.principal.type,
|
|
59
|
+
session.principal.id,
|
|
60
|
+
session.strategy,
|
|
61
|
+
session.status,
|
|
62
|
+
session.fileName ?? null,
|
|
63
|
+
session.mediaType ?? null,
|
|
64
|
+
session.logicalPath ?? null,
|
|
65
|
+
session.expectedSizeBytes ?? null,
|
|
66
|
+
session.expectedDigest ?? null,
|
|
67
|
+
...mutableColumns(session)
|
|
68
|
+
)
|
|
69
|
+
return true
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (isUniqueConstraintError(error)) return false
|
|
72
|
+
throw error
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async get(uploadId: string): Promise<FileUploadSession | null> {
|
|
77
|
+
const row = this.db.query(`${SELECT_SESSION} WHERE id = ?`).get(uploadId)
|
|
78
|
+
return row ? parseFileUploadSessionRow(row as Record<string, unknown>) : null
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async update(session: FileUploadSession): Promise<void> {
|
|
82
|
+
this.db
|
|
83
|
+
.query(
|
|
84
|
+
`UPDATE file_upload_sessions
|
|
85
|
+
SET status = ?, provider_upload = ?, signed_parts = ?, file_ref = ?, created_at = ?,
|
|
86
|
+
expires_at = ?, completed_at = ?, aborted_at = ?, reap_at = ?
|
|
87
|
+
WHERE id = ?`
|
|
88
|
+
)
|
|
89
|
+
.run(session.status, ...mutableColumns(session), session.id)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async listAbandoned(
|
|
93
|
+
input: ListAbandonedFileUploadSessionsInput
|
|
94
|
+
): Promise<readonly FileUploadSession[]> {
|
|
95
|
+
const rows = this.db
|
|
96
|
+
.query(
|
|
97
|
+
`${SELECT_SESSION}
|
|
98
|
+
WHERE project_id = ? AND status = 'pending' AND provider_upload IS NOT NULL
|
|
99
|
+
AND expires_at <= ?
|
|
100
|
+
ORDER BY expires_at, id
|
|
101
|
+
LIMIT ?`
|
|
102
|
+
)
|
|
103
|
+
.all(input.projectId, input.now.toISOString(), input.limit)
|
|
104
|
+
return rows.map((row) => parseFileUploadSessionRow(row as Record<string, unknown>))
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async deleteReapable(now: Date): Promise<number> {
|
|
108
|
+
return this.db
|
|
109
|
+
.query("DELETE FROM file_upload_sessions WHERE reap_at IS NOT NULL AND reap_at <= ?")
|
|
110
|
+
.run(now.toISOString()).changes
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const SELECT_SESSION = `
|
|
115
|
+
SELECT id, project_id AS projectId, owner_principal_type AS ownerPrincipalType,
|
|
116
|
+
owner_principal_id AS ownerPrincipalId, strategy, status,
|
|
117
|
+
file_name AS fileName, media_type AS mediaType, logical_path AS logicalPath,
|
|
118
|
+
expected_size_bytes AS expectedSizeBytes, expected_digest AS expectedDigest,
|
|
119
|
+
provider_upload AS providerUpload, signed_parts AS signedParts, file_ref AS fileRef,
|
|
120
|
+
created_at AS createdAt, expires_at AS expiresAt, completed_at AS completedAt,
|
|
121
|
+
aborted_at AS abortedAt
|
|
122
|
+
FROM file_upload_sessions`
|
|
123
|
+
|
|
124
|
+
/** Columns a transition may change, in insert/update order after `expected_digest`/`status`. */
|
|
125
|
+
function mutableColumns(session: FileUploadSession) {
|
|
126
|
+
return [
|
|
127
|
+
session.providerUpload === undefined ? null : JSON.stringify(session.providerUpload),
|
|
128
|
+
JSON.stringify(session.signedParts),
|
|
129
|
+
session.fileRef === undefined ? null : JSON.stringify(session.fileRef),
|
|
130
|
+
session.createdAt.toISOString(),
|
|
131
|
+
session.expiresAt.toISOString(),
|
|
132
|
+
session.completedAt?.toISOString() ?? null,
|
|
133
|
+
session.abortedAt?.toISOString() ?? null,
|
|
134
|
+
fileUploadSessionReapAt(session)?.toISOString() ?? null,
|
|
135
|
+
] as const
|
|
136
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -39,6 +39,7 @@ import { SqliteAiUsageStorage } from "./ai-usage-storage"
|
|
|
39
39
|
import { SqliteAuthStorage } from "./auth-storage"
|
|
40
40
|
import { SqliteConnectorConnectionStorage } from "./connector-connection-storage"
|
|
41
41
|
import { SqliteExecutionStorage } from "./execution-storage"
|
|
42
|
+
import { SqliteFileUploadSessionStorage } from "./file-upload-session-storage"
|
|
42
43
|
import {
|
|
43
44
|
createSqliteStorageMigrators,
|
|
44
45
|
installFreshSqliteSchema,
|
|
@@ -106,6 +107,7 @@ export class SqliteStorage implements MigrationCapableStorage {
|
|
|
106
107
|
readonly shareGrants: ShareGrantStorage
|
|
107
108
|
readonly shareSessions: ShareSessionStorage
|
|
108
109
|
readonly connectorConnections: SqliteConnectorConnectionStorage
|
|
110
|
+
readonly fileUploadSessions: SqliteFileUploadSessionStorage
|
|
109
111
|
readonly migrators: readonly StorageMigrator[]
|
|
110
112
|
|
|
111
113
|
private readonly connection: SqliteStoreConnection
|
|
@@ -172,6 +174,7 @@ export class SqliteStorage implements MigrationCapableStorage {
|
|
|
172
174
|
this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope)
|
|
173
175
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
|
|
174
176
|
this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope)
|
|
177
|
+
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
|
|
175
178
|
this.migrators = options.path ? createSqliteStorageMigrators(options.path) : []
|
|
176
179
|
registerSqliteStorageTestingAdapter(this, (durationMs) =>
|
|
177
180
|
stores.connectorConnections.advanceTimeForTesting(durationMs)
|
|
@@ -333,6 +336,7 @@ function createSqliteStores(
|
|
|
333
336
|
shareGrants,
|
|
334
337
|
shareSessions,
|
|
335
338
|
connectorConnections: new SqliteConnectorConnectionStorage(connection),
|
|
339
|
+
fileUploadSessions: new SqliteFileUploadSessionStorage(connection),
|
|
336
340
|
}
|
|
337
341
|
}
|
|
338
342
|
|
|
@@ -357,6 +361,7 @@ interface SqliteStoreSet {
|
|
|
357
361
|
readonly shareGrants: SqliteShareGrantStorage
|
|
358
362
|
readonly shareSessions: SqliteShareSessionStorage
|
|
359
363
|
readonly connectorConnections: SqliteConnectorConnectionStorage
|
|
364
|
+
readonly fileUploadSessions: SqliteFileUploadSessionStorage
|
|
360
365
|
}
|
|
361
366
|
|
|
362
367
|
export { migrateSqliteStorage } from "./migrations"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
-- Intervention actors were stored as { principalType, principalId }. Rewrite them to the
|
|
2
|
+
-- canonical Principal shape { type, id } used by every other stored principal.
|
|
3
|
+
UPDATE workflow_interventions
|
|
4
|
+
SET submitted_by = json_object(
|
|
5
|
+
'type', json_extract(submitted_by, '$.principalType'),
|
|
6
|
+
'id', json_extract(submitted_by, '$.principalId')
|
|
7
|
+
)
|
|
8
|
+
WHERE json_extract(submitted_by, '$.principalType') IS NOT NULL;
|
|
9
|
+
|
|
10
|
+
UPDATE workflow_interventions
|
|
11
|
+
SET cancelled_by = json_object(
|
|
12
|
+
'type', json_extract(cancelled_by, '$.principalType'),
|
|
13
|
+
'id', json_extract(cancelled_by, '$.principalId')
|
|
14
|
+
)
|
|
15
|
+
WHERE json_extract(cancelled_by, '$.principalType') IS NOT NULL;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
-- Commits and their events name who requested the write and what ran it, replacing the
|
|
2
|
+
-- authority-only `actor`. Both are copied from the commit's execution, which stays canonical and
|
|
3
|
+
-- is immutable, so the backfill is exact. SQLite cannot add a NOT NULL column without a default;
|
|
4
|
+
-- every insert supplies `executor`, and the UPDATE below replaces the placeholder on every row.
|
|
5
|
+
ALTER TABLE ontology_commits ADD COLUMN requested_by TEXT
|
|
6
|
+
CHECK (requested_by IS NULL OR json_valid(requested_by));
|
|
7
|
+
ALTER TABLE ontology_commits ADD COLUMN executor TEXT NOT NULL DEFAULT '{}'
|
|
8
|
+
CHECK (json_valid(executor));
|
|
9
|
+
|
|
10
|
+
UPDATE ontology_commits SET
|
|
11
|
+
requested_by = (
|
|
12
|
+
SELECT CASE
|
|
13
|
+
WHEN e.requested_by_user_id IS NOT NULL
|
|
14
|
+
THEN json_object('id', e.requested_by_user_id, 'type', 'user')
|
|
15
|
+
WHEN e.requested_by_service_account_id IS NOT NULL
|
|
16
|
+
THEN json_object('id', e.requested_by_service_account_id, 'type', 'serviceAccount')
|
|
17
|
+
END
|
|
18
|
+
FROM executions e
|
|
19
|
+
WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
|
|
20
|
+
),
|
|
21
|
+
executor = (
|
|
22
|
+
SELECT CASE e.executor_kind
|
|
23
|
+
WHEN 'request' THEN json_object('requestId', e.executor_id, 'type', 'request')
|
|
24
|
+
WHEN 'agent' THEN json_object('runId', e.executor_id, 'type', 'agent')
|
|
25
|
+
WHEN 'kernel' THEN json_object(
|
|
26
|
+
'operation',
|
|
27
|
+
CASE e.authority_kernel_operation
|
|
28
|
+
WHEN 'ontology.recover'
|
|
29
|
+
THEN json_object('recoveryId', e.executor_id, 'type', 'ontology.recover')
|
|
30
|
+
ELSE json_object('indexingId', e.executor_id, 'type', 'ontology.indexVectors')
|
|
31
|
+
END,
|
|
32
|
+
'type', 'kernel'
|
|
33
|
+
)
|
|
34
|
+
ELSE json_object(
|
|
35
|
+
'id', e.authority_primitive_id,
|
|
36
|
+
'kind', e.executor_kind,
|
|
37
|
+
'runId', e.executor_id,
|
|
38
|
+
'type', 'primitive'
|
|
39
|
+
)
|
|
40
|
+
END
|
|
41
|
+
FROM executions e
|
|
42
|
+
WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
-- Events not yet published would otherwise reach consumers in the retired shape.
|
|
46
|
+
UPDATE ontology_outbox SET envelope = (
|
|
47
|
+
SELECT json_remove(
|
|
48
|
+
CASE
|
|
49
|
+
WHEN c.requested_by IS NULL
|
|
50
|
+
THEN json_set(ontology_outbox.envelope, '$.executor', json(c.executor))
|
|
51
|
+
ELSE json_set(
|
|
52
|
+
ontology_outbox.envelope,
|
|
53
|
+
'$.executor', json(c.executor),
|
|
54
|
+
'$.requestedBy', json(c.requested_by)
|
|
55
|
+
)
|
|
56
|
+
END,
|
|
57
|
+
'$.actor'
|
|
58
|
+
)
|
|
59
|
+
FROM ontology_commits c
|
|
60
|
+
WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
|
|
61
|
+
)
|
|
62
|
+
WHERE EXISTS (
|
|
63
|
+
SELECT 1 FROM ontology_commits c
|
|
64
|
+
WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
ALTER TABLE ontology_commits DROP COLUMN actor;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
-- Durable staged-upload sessions, so an upload survives restarts and spans API replicas.
|
|
2
|
+
-- Timestamps are ISO-8601 UTC text and compare lexicographically.
|
|
3
|
+
CREATE TABLE file_upload_sessions (
|
|
4
|
+
id TEXT PRIMARY KEY CHECK (length(trim(id)) > 0),
|
|
5
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
6
|
+
owner_principal_type TEXT NOT NULL
|
|
7
|
+
CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
8
|
+
owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
|
|
9
|
+
strategy TEXT NOT NULL CHECK (strategy IN ('server', 'direct-put', 'multipart')),
|
|
10
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'aborted')),
|
|
11
|
+
file_name TEXT,
|
|
12
|
+
media_type TEXT,
|
|
13
|
+
logical_path TEXT,
|
|
14
|
+
expected_size_bytes INTEGER CHECK (expected_size_bytes IS NULL OR expected_size_bytes >= 0),
|
|
15
|
+
expected_digest TEXT,
|
|
16
|
+
provider_upload TEXT CHECK (provider_upload IS NULL OR json_type(provider_upload) = 'object'),
|
|
17
|
+
signed_parts TEXT NOT NULL CHECK (json_type(signed_parts) = 'array'),
|
|
18
|
+
file_ref TEXT CHECK (file_ref IS NULL OR json_type(file_ref) = 'object'),
|
|
19
|
+
created_at TEXT NOT NULL,
|
|
20
|
+
expires_at TEXT NOT NULL,
|
|
21
|
+
completed_at TEXT,
|
|
22
|
+
aborted_at TEXT,
|
|
23
|
+
-- Derived by core (fileUploadSessionReapAt); NULL while an abandoned upload awaits its abort.
|
|
24
|
+
reap_at TEXT,
|
|
25
|
+
CHECK ((status = 'completed') = (completed_at IS NOT NULL)),
|
|
26
|
+
CHECK ((status = 'aborted') = (aborted_at IS NOT NULL))
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
CREATE INDEX idx_file_upload_sessions_reap
|
|
30
|
+
ON file_upload_sessions (reap_at)
|
|
31
|
+
WHERE reap_at IS NOT NULL;
|
|
32
|
+
|
|
33
|
+
CREATE INDEX idx_file_upload_sessions_abandoned
|
|
34
|
+
ON file_upload_sessions (project_id, expires_at)
|
|
35
|
+
WHERE status = 'pending' AND provider_upload IS NOT NULL;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Native clients, such as the CLI, hold their session as bearer tokens: a
|
|
2
|
+
-- short-lived access token, whose hash is `token_hash`, and a rotating refresh token. Browser
|
|
3
|
+
-- sessions leave these columns NULL; `refresh_token_hash IS NULL` is what marks a cookie session.
|
|
4
|
+
ALTER TABLE auth_sessions ADD COLUMN client_name TEXT;
|
|
5
|
+
ALTER TABLE auth_sessions ADD COLUMN access_expires_at TEXT;
|
|
6
|
+
ALTER TABLE auth_sessions ADD COLUMN refresh_token_hash TEXT;
|
|
7
|
+
ALTER TABLE auth_sessions ADD COLUMN previous_refresh_token_hash TEXT;
|
|
8
|
+
ALTER TABLE auth_sessions ADD COLUMN refreshed_at TEXT;
|
|
9
|
+
|
|
10
|
+
-- The device flow now starts a session instead of issuing a personal access token.
|
|
11
|
+
ALTER TABLE auth_device_authorizations DROP COLUMN token_name;
|
|
12
|
+
ALTER TABLE auth_device_authorizations DROP COLUMN token_expires_at;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
-- A membership the identity provider owns is recorded with source 'directory' and synced to the
|
|
2
|
+
-- provider's groups on every sign-in. Service accounts never sign in, so their table is unchanged.
|
|
3
|
+
-- SQLite cannot change a CHECK constraint in place, so the table is rebuilt.
|
|
4
|
+
CREATE TABLE auth_group_memberships_next (
|
|
5
|
+
project_id TEXT NOT NULL,
|
|
6
|
+
user_id TEXT NOT NULL,
|
|
7
|
+
group_id TEXT NOT NULL,
|
|
8
|
+
source TEXT NOT NULL CHECK (source IN ('invitation', 'manual', 'agent', 'directory')),
|
|
9
|
+
created_at TEXT NOT NULL,
|
|
10
|
+
PRIMARY KEY (project_id, user_id, group_id)
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
INSERT INTO auth_group_memberships_next (project_id, user_id, group_id, source, created_at)
|
|
14
|
+
SELECT project_id, user_id, group_id, source, created_at
|
|
15
|
+
FROM auth_group_memberships;
|
|
16
|
+
|
|
17
|
+
DROP TABLE auth_group_memberships;
|
|
18
|
+
ALTER TABLE auth_group_memberships_next RENAME TO auth_group_memberships;
|
|
19
|
+
|
|
20
|
+
CREATE INDEX idx_auth_group_memberships_group
|
|
21
|
+
ON auth_group_memberships(project_id, group_id);
|
package/src/migrations.ts
CHANGED
|
@@ -105,6 +105,16 @@ import vectorBatchingSql from "./migrations/044-vector-batching.sql" with { type
|
|
|
105
105
|
import agentThreadSandboxStateSql from "./migrations/045-agent-thread-sandbox-state.sql" with {
|
|
106
106
|
type: "text",
|
|
107
107
|
}
|
|
108
|
+
import workflowInterventionPrincipalsSql from "./migrations/046-workflow-intervention-principals.sql" with {
|
|
109
|
+
type: "text",
|
|
110
|
+
}
|
|
111
|
+
import ontologyCommitAttributionSql from "./migrations/047-ontology-commit-attribution.sql" with {
|
|
112
|
+
type: "text",
|
|
113
|
+
}
|
|
114
|
+
import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type: "text" }
|
|
115
|
+
import directoryGroupMembershipsSql from "./migrations/050-directory-group-memberships.sql" with {
|
|
116
|
+
type: "text",
|
|
117
|
+
}
|
|
108
118
|
|
|
109
119
|
const MIGRATIONS_TABLE_SQL = `
|
|
110
120
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -205,6 +215,11 @@ export const sqliteStorageMigrations = defineMigrations({
|
|
|
205
215
|
sqliteSql("043-vector-indexing", vectorIndexingSql),
|
|
206
216
|
sqliteSql("044-vector-batching", vectorBatchingSql),
|
|
207
217
|
sqliteSql("045-agent-thread-sandbox-state", agentThreadSandboxStateSql),
|
|
218
|
+
sqliteSql("046-workflow-intervention-principals", workflowInterventionPrincipalsSql),
|
|
219
|
+
sqliteSql("047-ontology-commit-attribution", ontologyCommitAttributionSql),
|
|
220
|
+
sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
|
|
221
|
+
sqliteSql("049-native-sessions", nativeSessionsSql),
|
|
222
|
+
sqliteSql("050-directory-group-memberships", directoryGroupMembershipsSql),
|
|
208
223
|
],
|
|
209
224
|
})
|
|
210
225
|
|
|
@@ -428,3 +443,4 @@ import executionRequesterGroupsSql from "./migrations/037-execution-requester-gr
|
|
|
428
443
|
import connectorOptionalPkceSql from "./migrations/039-connector-optional-pkce.sql" with {
|
|
429
444
|
type: "text",
|
|
430
445
|
}
|
|
446
|
+
import fileUploadSessionsSql from "./migrations/048-file-upload-sessions.sql" with { type: "text" }
|
|
@@ -963,9 +963,13 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
963
963
|
return negatePredicate(item)
|
|
964
964
|
}
|
|
965
965
|
case "eq":
|
|
966
|
-
return
|
|
966
|
+
return predicate.scalarKind === "userRef"
|
|
967
|
+
? compileUserRefInPredicate(predicate.propertyId, [predicate.value])
|
|
968
|
+
: compileEqualityPredicate(predicate.propertyId, "eq", predicate.value)
|
|
967
969
|
case "neq":
|
|
968
|
-
return
|
|
970
|
+
return predicate.scalarKind === "userRef"
|
|
971
|
+
? negatePredicate(compileUserRefInPredicate(predicate.propertyId, [predicate.value]))
|
|
972
|
+
: compileEqualityPredicate(predicate.propertyId, "neq", predicate.value)
|
|
969
973
|
case "lt":
|
|
970
974
|
case "lte":
|
|
971
975
|
case "gt":
|
|
@@ -977,17 +981,68 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
977
981
|
}
|
|
978
982
|
}
|
|
979
983
|
case "in":
|
|
980
|
-
return
|
|
984
|
+
return predicate.scalarKind === "userRef"
|
|
985
|
+
? compileUserRefInPredicate(predicate.propertyId, predicate.values)
|
|
986
|
+
: compileInPredicate(predicate.propertyId, predicate.values)
|
|
981
987
|
case "exists": {
|
|
982
988
|
const sql = `${jsonTypeExpression()} IS NOT NULL`
|
|
983
989
|
const item: CompiledPredicate = { sql, args: [jsonPath(predicate.propertyId)] }
|
|
984
990
|
return predicate.value ? item : negatePredicate(item)
|
|
985
991
|
}
|
|
986
992
|
case "contains":
|
|
987
|
-
return
|
|
993
|
+
return predicate.scalarKind === "userRef"
|
|
994
|
+
? compileUserRefContainsPredicate(predicate.propertyId, predicate.value)
|
|
995
|
+
: compileContainsPredicate(predicate.propertyId, predicate.value)
|
|
988
996
|
}
|
|
989
997
|
}
|
|
990
998
|
|
|
999
|
+
/**
|
|
1000
|
+
* User references are JSON objects, so they match on both fields rather than on serialized text,
|
|
1001
|
+
* which would depend on key order.
|
|
1002
|
+
*/
|
|
1003
|
+
function compileUserRefInPredicate(
|
|
1004
|
+
propertyId: string,
|
|
1005
|
+
values: readonly unknown[]
|
|
1006
|
+
): CompiledPredicate {
|
|
1007
|
+
const path = jsonPath(propertyId)
|
|
1008
|
+
const ids = values.flatMap((value) => {
|
|
1009
|
+
const id = userRefId(value)
|
|
1010
|
+
return id === undefined ? [] : [id]
|
|
1011
|
+
})
|
|
1012
|
+
const clauses: string[] = []
|
|
1013
|
+
const args: SqliteValue[] = []
|
|
1014
|
+
|
|
1015
|
+
if (values.includes(null)) {
|
|
1016
|
+
clauses.push(`${jsonTypeExpression()} = 'null'`)
|
|
1017
|
+
args.push(path)
|
|
1018
|
+
}
|
|
1019
|
+
if (ids.length > 0) {
|
|
1020
|
+
clauses.push(
|
|
1021
|
+
`(${jsonValueExpression()} = 'user' AND ${jsonValueExpression()} IN (${ids.map(() => "?").join(", ")}))`
|
|
1022
|
+
)
|
|
1023
|
+
args.push(`${path}.type`, `${path}.id`, ...ids)
|
|
1024
|
+
}
|
|
1025
|
+
if (clauses.length === 0) return { sql: "0 = 1", args: [] }
|
|
1026
|
+
|
|
1027
|
+
return { sql: `(${clauses.join(" OR ")})`, args }
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
function compileUserRefContainsPredicate(propertyId: string, value: unknown): CompiledPredicate {
|
|
1031
|
+
const id = userRefId(value)
|
|
1032
|
+
if (id === undefined) return { sql: "0 = 1", args: [] }
|
|
1033
|
+
const path = jsonPath(propertyId)
|
|
1034
|
+
return {
|
|
1035
|
+
sql: `(${jsonTypeExpression()} = 'array' AND EXISTS (SELECT 1 FROM json_each(properties, ?) AS item WHERE item.type = 'object' AND json_extract(item.value, '$.type') = 'user' AND json_extract(item.value, '$.id') = ?))`,
|
|
1036
|
+
args: [path, path, id],
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
function userRefId(value: unknown): string | undefined {
|
|
1041
|
+
if (typeof value !== "object" || value === null) return undefined
|
|
1042
|
+
const { type, id } = value as { type?: unknown; id?: unknown }
|
|
1043
|
+
return type === "user" && typeof id === "string" ? id : undefined
|
|
1044
|
+
}
|
|
1045
|
+
|
|
991
1046
|
function negatePredicate(predicate: CompiledPredicate): CompiledPredicate {
|
|
992
1047
|
return {
|
|
993
1048
|
sql: `(NOT COALESCE((${predicate.sql}), 0))`,
|
package/src/objects/reader.ts
CHANGED
|
@@ -512,6 +512,10 @@ function sqliteFacetValue(valueType: string | null, value: unknown): unknown {
|
|
|
512
512
|
if (valueType === "true") return true
|
|
513
513
|
if (valueType === "false") return false
|
|
514
514
|
if (valueType === "null") return null
|
|
515
|
+
// json_extract returns structured values, such as user references, as JSON text.
|
|
516
|
+
if ((valueType === "object" || valueType === "array") && typeof value === "string") {
|
|
517
|
+
return JSON.parse(value)
|
|
518
|
+
}
|
|
515
519
|
return value
|
|
516
520
|
}
|
|
517
521
|
|
package/src/objects/storage.ts
CHANGED
|
@@ -95,6 +95,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
95
95
|
decimal: { equality: true },
|
|
96
96
|
date: { equality: true, ordering: true },
|
|
97
97
|
timestamp: { equality: true, ordering: true },
|
|
98
|
+
userRef: { equality: true },
|
|
98
99
|
},
|
|
99
100
|
limits: {
|
|
100
101
|
totalCount: true,
|
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
|
-
import type {
|
|
3
|
-
ExpectedLinkRevision,
|
|
4
|
-
ExpectedObjectRevision,
|
|
5
|
-
} from "@sixb/core/internal/materialization"
|
|
6
2
|
import {
|
|
7
3
|
assertPinnedDatasetWatermark,
|
|
8
4
|
linkRefKey,
|
|
@@ -16,6 +12,9 @@ import {
|
|
|
16
12
|
telemetryPointSortKey,
|
|
17
13
|
} from "@sixb/core/internal/materialization"
|
|
18
14
|
import {
|
|
15
|
+
assertExpectedLinkRevision,
|
|
16
|
+
assertExpectedLinkScopeRevision,
|
|
17
|
+
assertExpectedObjectRevision,
|
|
19
18
|
assertMaterializationFinalizationCorrelation,
|
|
20
19
|
assertMaterializationHeader,
|
|
21
20
|
assertMaterializationLaneCompletion,
|
|
@@ -100,16 +99,15 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
100
99
|
const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId)
|
|
101
100
|
for (const expected of input.expected.sources)
|
|
102
101
|
this.assertSource(expected, input.commit.projectId)
|
|
103
|
-
for (const expected of input.expected.objects)
|
|
104
|
-
|
|
102
|
+
for (const expected of input.expected.objects) {
|
|
103
|
+
assertExpectedObjectRevision(reader.effectiveObjectRevision(expected.ref), expected)
|
|
104
|
+
}
|
|
105
|
+
for (const expected of input.expected.links) {
|
|
106
|
+
assertExpectedLinkRevision(reader.effectiveLinkLastCommit(expected.ref), expected)
|
|
107
|
+
}
|
|
105
108
|
const linkScopeRevisions = reader.linkScopeRevisions(input.expected.linkScopes)
|
|
106
109
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
107
|
-
|
|
108
|
-
throw new MaterializationConflictError(
|
|
109
|
-
"effective-state",
|
|
110
|
-
`Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`
|
|
111
|
-
)
|
|
112
|
-
}
|
|
110
|
+
assertExpectedLinkScopeRevision(linkScopeRevisions[index]?.fingerprint, expected)
|
|
113
111
|
}
|
|
114
112
|
for (const expected of input.expected.points) {
|
|
115
113
|
const point = reader.exactPoint(expected.series, expected.at)
|
|
@@ -449,50 +447,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
449
447
|
}
|
|
450
448
|
}
|
|
451
449
|
|
|
452
|
-
private assertObject(
|
|
453
|
-
reader: SqliteMaterializationStateReader,
|
|
454
|
-
expected: ExpectedObjectRevision
|
|
455
|
-
): void {
|
|
456
|
-
const row = reader.effectiveObjectRevision(expected.ref)
|
|
457
|
-
if (!expected.exists) {
|
|
458
|
-
if (row) {
|
|
459
|
-
throw new MaterializationConflictError(
|
|
460
|
-
"effective-state",
|
|
461
|
-
`Expected object ${objectRefKey(expected.ref)} to be absent.`
|
|
462
|
-
)
|
|
463
|
-
}
|
|
464
|
-
return
|
|
465
|
-
}
|
|
466
|
-
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
467
|
-
throw new MaterializationConflictError(
|
|
468
|
-
"effective-state",
|
|
469
|
-
`Expected object ${objectRefKey(expected.ref)} changed.`
|
|
470
|
-
)
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
private assertLink(
|
|
475
|
-
reader: SqliteMaterializationStateReader,
|
|
476
|
-
expected: ExpectedLinkRevision
|
|
477
|
-
): void {
|
|
478
|
-
const lastCommitId = reader.effectiveLinkLastCommit(expected.ref)
|
|
479
|
-
if (!expected.exists) {
|
|
480
|
-
if (lastCommitId !== undefined) {
|
|
481
|
-
throw new MaterializationConflictError(
|
|
482
|
-
"effective-state",
|
|
483
|
-
`Expected link ${linkRefKey(expected.ref)} to be absent.`
|
|
484
|
-
)
|
|
485
|
-
}
|
|
486
|
-
return
|
|
487
|
-
}
|
|
488
|
-
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
489
|
-
throw new MaterializationConflictError(
|
|
490
|
-
"effective-state",
|
|
491
|
-
`Expected link ${linkRefKey(expected.ref)} changed.`
|
|
492
|
-
)
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
|
|
496
450
|
private async assertFinalization(
|
|
497
451
|
session: SqliteMaterializationSessionState,
|
|
498
452
|
input: FinalizeMaterializationInput
|
|
@@ -719,10 +673,12 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
719
673
|
`
|
|
720
674
|
INSERT INTO ontology_commits (
|
|
721
675
|
project_id, id, idempotency_key, request_hash, execution_id,
|
|
722
|
-
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
676
|
+
origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
|
|
723
677
|
ontology_revision, projection_revision, ownership_hash,
|
|
724
678
|
intent, result, committed_at
|
|
725
|
-
) VALUES (
|
|
679
|
+
) VALUES (
|
|
680
|
+
?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
|
|
681
|
+
)
|
|
726
682
|
`
|
|
727
683
|
)
|
|
728
684
|
.run(
|
|
@@ -735,7 +691,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
735
691
|
origin.runId,
|
|
736
692
|
origin.batchOrdinal,
|
|
737
693
|
canonicalJson(commit.origin),
|
|
738
|
-
commit.
|
|
694
|
+
commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy),
|
|
695
|
+
canonicalJson(commit.executor),
|
|
739
696
|
commit.ontologyRevision,
|
|
740
697
|
commit.projectionRevision ?? null,
|
|
741
698
|
commit.ownershipHash ?? null,
|
|
@@ -37,7 +37,8 @@ export interface SqliteOntologyCommitRow {
|
|
|
37
37
|
readonly origin_run_id: string | null
|
|
38
38
|
readonly origin_batch_ordinal: number | null
|
|
39
39
|
readonly origin: string
|
|
40
|
-
readonly
|
|
40
|
+
readonly requested_by: string | null
|
|
41
|
+
readonly executor: string
|
|
41
42
|
readonly ontology_revision: string
|
|
42
43
|
readonly projection_revision: string | null
|
|
43
44
|
readonly ownership_hash: string | null
|
|
@@ -162,9 +163,12 @@ export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord
|
|
|
162
163
|
requestHash: row.request_hash,
|
|
163
164
|
executionId: row.execution_id,
|
|
164
165
|
origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
|
|
165
|
-
...(row.
|
|
166
|
+
...(row.requested_by === null
|
|
166
167
|
? {}
|
|
167
|
-
: {
|
|
168
|
+
: {
|
|
169
|
+
requestedBy: parseJson<NonNullable<OntologyCommitWrite["requestedBy"]>>(row.requested_by),
|
|
170
|
+
}),
|
|
171
|
+
executor: parseJson<OntologyCommitWrite["executor"]>(row.executor),
|
|
168
172
|
ontologyRevision: row.ontology_revision,
|
|
169
173
|
...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
|
|
170
174
|
...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
|