or3-provider-sqlite 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -14
- package/dist/module.json +1 -1
- package/dist/runtime/server/admin/stores/sqlite-store.js +16 -0
- package/dist/runtime/server/background-jobs/sqlite-provider.d.ts +2 -0
- package/dist/runtime/server/background-jobs/sqlite-provider.js +68 -7
- package/dist/runtime/server/plugin-connections/sqlite-connection-store.d.ts +7 -0
- package/dist/runtime/server/plugin-connections/sqlite-connection-store.js +209 -0
- package/dist/runtime/server/plugins/register.js +16 -1
- package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.js +71 -18
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ runtime.
|
|
|
32
32
|
| Runtime | `OR3_SQLITE_DRIVER` | Required configuration | Extra install |
|
|
33
33
|
|---|---|---|---|
|
|
34
34
|
| Local Node (default) | `better-sqlite3` | `OR3_SQLITE_DB_PATH` | `better-sqlite3` |
|
|
35
|
-
| Bun | `bun` | `OR3_SQLITE_DB_PATH` | None; uses built-in `bun:sqlite` |
|
|
35
|
+
| Bun | `bun` | `OR3_SQLITE_DB_PATH` | None for sync; uses built-in `bun:sqlite` |
|
|
36
36
|
| Turso/libSQL | `turso` | `OR3_SQLITE_TURSO_URL`, `OR3_SQLITE_TURSO_AUTH_TOKEN` | `libsql` |
|
|
37
37
|
| Cloudflare D1 | `d1` | `OR3_SQLITE_D1_BINDING` (defaults to `DB`) | None; uses the Worker binding |
|
|
38
38
|
|
|
@@ -51,6 +51,16 @@ OR3_SQLITE_DRIVER=bun
|
|
|
51
51
|
OR3_SQLITE_DB_PATH=/data/or3-sync.db
|
|
52
52
|
```
|
|
53
53
|
|
|
54
|
+
OR3 Chat's source launcher (`bun run dev:ssr`) selects Bun when this provider
|
|
55
|
+
uses the Bun driver, including when only the auth workspace store is active.
|
|
56
|
+
Use the Bun version pinned in the host's `package.json`. The launcher checks
|
|
57
|
+
the built-in driver and required native bindings before Nuxt starts. Run a
|
|
58
|
+
built Bun-configured server with `bun .output/server/index.mjs`.
|
|
59
|
+
|
|
60
|
+
Basic Auth still uses a separate `better-sqlite3` database; its native binding
|
|
61
|
+
must also be compatible with Bun. The managed Cloud image uses Node and the
|
|
62
|
+
default `better-sqlite3` sync driver.
|
|
63
|
+
|
|
54
64
|
### Turso/libSQL
|
|
55
65
|
|
|
56
66
|
```bash
|
|
@@ -117,19 +127,33 @@ OR3_CONNECT_PROVIDER=sqlite
|
|
|
117
127
|
|
|
118
128
|
On server startup, the Nitro plugin:
|
|
119
129
|
|
|
120
|
-
1.
|
|
121
|
-
2.
|
|
122
|
-
3. Registers `
|
|
123
|
-
4. Registers `
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
7. Registers `RateLimitProvider` with ID `sqlite` and the sync admin adapter
|
|
128
|
-
8. Registers a durable `BackgroundJobProvider` with ID `sqlite`, so active chat
|
|
130
|
+
1. Registers `AuthWorkspaceStore` with ID `sqlite`
|
|
131
|
+
2. Registers `SyncGatewayAdapter` when SQLite conversation transfer is enabled
|
|
132
|
+
3. Registers `ConnectStore` when SQLite Connect persistence is selected
|
|
133
|
+
4. Registers `WebhookStore` on runtimes other than D1, and the admin store
|
|
134
|
+
provider (D1 declares server-side admin capabilities unavailable)
|
|
135
|
+
5. Registers `RateLimitProvider` with ID `sqlite` and the sync admin adapter
|
|
136
|
+
6. Registers a durable `BackgroundJobProvider` with ID `sqlite`, so active chat
|
|
129
137
|
and workflow jobs remain addressable across Nitro HMR/module reloads
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
138
|
+
7. Initializes the database and runs schema migrations eagerly on local
|
|
139
|
+
runtimes, or on the first Worker request for D1
|
|
140
|
+
|
|
141
|
+
Registration is skipped when `auth.enabled` is `false`, or when SQLite is not
|
|
142
|
+
selected for account workspaces, Connect or background jobs. Account workspaces
|
|
143
|
+
use `sync.provider` even with `sync.enabled=false`; disabling conversation
|
|
144
|
+
transfer does not disable the workspace store. The sync gateway remains
|
|
145
|
+
unregistered while transfer is disabled.
|
|
146
|
+
|
|
147
|
+
Workspace-only account storage requires `or3-provider-sqlite@0.0.12` or later.
|
|
148
|
+
Version `0.0.11` skips workspace-store registration when sync, Connect and
|
|
149
|
+
background jobs are all disabled, causing Chat to exit before listening.
|
|
150
|
+
Upgrade the provider; conversation transfer can remain disabled.
|
|
151
|
+
|
|
152
|
+
SQLite background jobs also persist mixed-runtime tool handoffs. Browser-only
|
|
153
|
+
calls are atomically claimed and settled in `execution_json`; parked calls are
|
|
154
|
+
excluded from worker claims until their browser result makes the job runnable.
|
|
155
|
+
Parked handoffs have a separate global and per-user admission cap, so they do
|
|
156
|
+
not consume active worker slots or grow without bound.
|
|
133
157
|
|
|
134
158
|
### Schema
|
|
135
159
|
|
|
@@ -164,7 +188,7 @@ All tables use snake_case aligned with the sync wire format.
|
|
|
164
188
|
- **Retention safety**: tombstone and `change_log` GC is enabled only under the explicit `snapshot-v1` capability and deletes old revisions acknowledged by every registered device
|
|
165
189
|
- **Notifications**: push forces `payload.user_id` to the session user; foreign notification writes are rejected; pull/snapshot omit other users' rows
|
|
166
190
|
|
|
167
|
-
LWW conflict resolution: incoming wins when `clock` is higher, then when clocks are equal and `hlc` is lexicographically greater, then when `op_id` is greater. Tombstones use the same `(clock, hlc, op_id)` tuple. LWW losers return `applied: false` with the
|
|
191
|
+
LWW conflict resolution: incoming wins when `clock` is higher, then when clocks are equal and `hlc` is lexicographically greater, then when `op_id` is greater. Tombstones use the same `(clock, hlc, op_id)` tuple. LWW losers return `applied: false` with the current winner as either `{ kind: 'put', payload, revision }` or `{ kind: 'delete', revision, serverDeletedAt? }`. Idempotent `op_id` retries return `replayed: true` and, if superseded, the current winner. A replay does not allocate another server version.
|
|
168
192
|
|
|
169
193
|
Local, Bun, and Turso runtimes use `BEGIN IMMEDIATE` transactions. D1 uses its
|
|
170
194
|
native atomic batch API for grouped writes.
|
|
@@ -235,3 +259,17 @@ bun run build # build for distribution
|
|
|
235
259
|
`OR3_CONNECT_ENABLED=false` or switch to better-sqlite3, Bun, or Turso.
|
|
236
260
|
- **`OR3_SQLITE_DRIVER=bun requires Bun`** — the Bun driver only runs under
|
|
237
261
|
the Bun runtime with its built-in `bun:sqlite`.
|
|
262
|
+
|
|
263
|
+
### Atomic workspace settings
|
|
264
|
+
|
|
265
|
+
The workspace settings store implements `compareAndSet(workspaceId, key, expectedValue, nextValue)` atomically. A `null` expected value means the key must be absent; a conflict returns `false` without overwriting it. The host uses this for concurrent plugin setup saves, bulk plugin enablement and permission review, and persistent AI spend reservations. A provider without this operation cannot safely run a bulk plugin rollout. A slow plugin write may outlive the HTTP response: the host reports pending after eight seconds and keeps the plugin lock until the atomic operation settles.
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
### Testing local changes in OR3 Chat
|
|
269
|
+
|
|
270
|
+
With this repository beside `or3-chat`, run `bun install` here once, then
|
|
271
|
+
`bun run dev:ssr` from Chat. Chat's dev wrapper rebuilds the local provider and
|
|
272
|
+
prints its selected path; restart it after provider edits. Missing repositories
|
|
273
|
+
or failed builds fall back to installed packages with a warning.
|
|
274
|
+
`OR3_LOCAL_PROVIDERS=false` disables local selection. Production builds use the
|
|
275
|
+
installed package, so local development does not publish these changes.
|
package/dist/module.json
CHANGED
|
@@ -323,6 +323,22 @@ class SqliteWorkspaceSettingsStore {
|
|
|
323
323
|
})
|
|
324
324
|
).execute();
|
|
325
325
|
}
|
|
326
|
+
/** Atomically write only when the stored value still equals the expected value. */
|
|
327
|
+
async compareAndSet(workspaceId, key, expectedValue, nextValue) {
|
|
328
|
+
const now = nowEpoch();
|
|
329
|
+
if (expectedValue === null) {
|
|
330
|
+
const result2 = await this.db.insertInto("admin_workspace_settings").values({
|
|
331
|
+
id: globalThis.crypto.randomUUID(),
|
|
332
|
+
workspace_id: workspaceId,
|
|
333
|
+
key,
|
|
334
|
+
value: nextValue,
|
|
335
|
+
updated_at: now
|
|
336
|
+
}).onConflict((oc) => oc.columns(["workspace_id", "key"]).doNothing()).executeTakeFirst();
|
|
337
|
+
return Number(result2.numInsertedOrUpdatedRows ?? 0) === 1;
|
|
338
|
+
}
|
|
339
|
+
const result = await this.db.updateTable("admin_workspace_settings").set({ value: nextValue, updated_at: now }).where("workspace_id", "=", workspaceId).where("key", "=", key).where("value", "=", expectedValue).executeTakeFirst();
|
|
340
|
+
return Number(result.numUpdatedRows ?? 0) === 1;
|
|
341
|
+
}
|
|
326
342
|
}
|
|
327
343
|
class SqliteAdminUserStore {
|
|
328
344
|
get db() {
|
|
@@ -16,6 +16,8 @@ export declare class SqliteBackgroundJobProvider implements BackgroundJobProvide
|
|
|
16
16
|
claimNextJob(leaseOwner: string, now: number, leaseExpiresAt: number): Promise<BackgroundJob | null>;
|
|
17
17
|
renewJobLease(jobId: string, leaseOwner: string, _now: number, leaseExpiresAt: number): Promise<boolean>;
|
|
18
18
|
updateJobExecution(jobId: string, execution: BackgroundJobExecution, leaseOwner: string): Promise<boolean>;
|
|
19
|
+
claimClientToolCall(jobId: string, userId: string, callId: string, claimToken: string, claimExpiresAt: number): Promise<BackgroundJob | null>;
|
|
20
|
+
settleClientToolCall(jobId: string, userId: string, callId: string, claimToken: string, execution: BackgroundJobExecution, toolCalls: BackgroundJob['tool_calls']): Promise<boolean>;
|
|
19
21
|
saveTerminalSnapshot(jobId: string, snapshot: TerminalGenerationSnapshot, leaseOwner?: string): Promise<boolean>;
|
|
20
22
|
setHistoryPhase(jobId: string, phase: GenerationHistoryPhase, options?: {
|
|
21
23
|
from?: GenerationHistoryPhase[];
|
|
@@ -109,8 +109,14 @@ const INSERT_JOB_IF_CAPACITY = `INSERT INTO background_jobs (
|
|
|
109
109
|
chunks_received, started_at, last_activity_at, tool_calls_json,
|
|
110
110
|
workflow_state_json, execution_json, idempotency_key, attempts
|
|
111
111
|
) SELECT ?, ?, ?, ?, ?, ?, 'streaming', ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, 0
|
|
112
|
-
WHERE (SELECT COUNT(*) FROM background_jobs WHERE status = 'streaming'
|
|
113
|
-
|
|
112
|
+
WHERE (SELECT COUNT(*) FROM background_jobs WHERE status = 'streaming'
|
|
113
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NULL) < ?
|
|
114
|
+
AND (SELECT COUNT(*) FROM background_jobs WHERE status = 'streaming' AND user_id = ?
|
|
115
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NULL) < ?
|
|
116
|
+
AND (SELECT COUNT(*) FROM background_jobs WHERE status = 'streaming'
|
|
117
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NOT NULL) < ?
|
|
118
|
+
AND (SELECT COUNT(*) FROM background_jobs WHERE status = 'streaming' AND user_id = ?
|
|
119
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NOT NULL) < ?
|
|
114
120
|
AND NOT EXISTS (
|
|
115
121
|
SELECT 1 FROM background_admission_cancels
|
|
116
122
|
WHERE user_id = ? AND admission_id = ? AND expires_at > ?
|
|
@@ -144,6 +150,9 @@ export class SqliteBackgroundJobProvider {
|
|
|
144
150
|
config.maxConcurrentJobs,
|
|
145
151
|
params.userId,
|
|
146
152
|
config.maxConcurrentJobsPerUser,
|
|
153
|
+
config.maxConcurrentJobs,
|
|
154
|
+
params.userId,
|
|
155
|
+
config.maxConcurrentJobsPerUser,
|
|
147
156
|
params.userId,
|
|
148
157
|
params.idempotencyKey ?? "",
|
|
149
158
|
now
|
|
@@ -186,11 +195,14 @@ export class SqliteBackgroundJobProvider {
|
|
|
186
195
|
}
|
|
187
196
|
}
|
|
188
197
|
const active = raw.prepare(
|
|
189
|
-
`SELECT
|
|
190
|
-
SUM(CASE WHEN
|
|
198
|
+
`SELECT
|
|
199
|
+
SUM(CASE WHEN json_extract(execution_json, '$.clientToolCall') IS NULL THEN 1 ELSE 0 END) AS total,
|
|
200
|
+
SUM(CASE WHEN user_id = ? AND json_extract(execution_json, '$.clientToolCall') IS NULL THEN 1 ELSE 0 END) AS user_total,
|
|
201
|
+
SUM(CASE WHEN json_extract(execution_json, '$.clientToolCall') IS NOT NULL THEN 1 ELSE 0 END) AS waiting_total,
|
|
202
|
+
SUM(CASE WHEN user_id = ? AND json_extract(execution_json, '$.clientToolCall') IS NOT NULL THEN 1 ELSE 0 END) AS waiting_user_total
|
|
191
203
|
FROM background_jobs WHERE status = 'streaming'`
|
|
192
|
-
).get(params.userId);
|
|
193
|
-
if (active.total >= config.maxConcurrentJobs || (active.user_total ?? 0) >= config.maxConcurrentJobsPerUser) {
|
|
204
|
+
).get(params.userId, params.userId);
|
|
205
|
+
if ((active.total ?? 0) >= config.maxConcurrentJobs || (active.user_total ?? 0) >= config.maxConcurrentJobsPerUser || (active.waiting_total ?? 0) >= config.maxConcurrentJobs || (active.waiting_user_total ?? 0) >= config.maxConcurrentJobsPerUser) {
|
|
194
206
|
throw new Error(
|
|
195
207
|
"Maximum concurrent background jobs reached"
|
|
196
208
|
);
|
|
@@ -394,6 +406,7 @@ export class SqliteBackgroundJobProvider {
|
|
|
394
406
|
chunks_received = CASE WHEN attempts > 0 THEN 0 ELSE chunks_received END,
|
|
395
407
|
attempts = attempts + 1
|
|
396
408
|
WHERE id = ? AND status = 'streaming' AND execution_json IS NOT NULL
|
|
409
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NULL
|
|
397
410
|
AND COALESCE(history_phase, 'ready') = 'ready'
|
|
398
411
|
AND (lease_owner IS NULL OR lease_expires_at <= ?)
|
|
399
412
|
RETURNING *`,
|
|
@@ -422,6 +435,7 @@ export class SqliteBackgroundJobProvider {
|
|
|
422
435
|
WHERE id = (
|
|
423
436
|
SELECT id FROM background_jobs
|
|
424
437
|
WHERE status = 'streaming' AND execution_json IS NOT NULL
|
|
438
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NULL
|
|
425
439
|
AND COALESCE(history_phase, 'ready') = 'ready'
|
|
426
440
|
AND (lease_owner IS NULL OR lease_expires_at <= ?)
|
|
427
441
|
ORDER BY started_at ASC LIMIT 1
|
|
@@ -455,6 +469,52 @@ export class SqliteBackgroundJobProvider {
|
|
|
455
469
|
leaseOwner
|
|
456
470
|
) > 0;
|
|
457
471
|
}
|
|
472
|
+
async claimClientToolCall(jobId, userId, callId, claimToken, claimExpiresAt) {
|
|
473
|
+
const now = Date.now();
|
|
474
|
+
const row = await one(
|
|
475
|
+
`UPDATE background_jobs SET
|
|
476
|
+
execution_json = json_set(
|
|
477
|
+
execution_json,
|
|
478
|
+
'$.clientToolCall.claimToken', ?,
|
|
479
|
+
'$.clientToolCall.claimExpiresAt', ?
|
|
480
|
+
),
|
|
481
|
+
last_activity_at = ?
|
|
482
|
+
WHERE id = ? AND user_id = ? AND status = 'streaming'
|
|
483
|
+
AND json_extract(execution_json, '$.clientToolCall.callId') = ?
|
|
484
|
+
AND (
|
|
485
|
+
json_extract(execution_json, '$.clientToolCall.claimToken') IS NULL
|
|
486
|
+
OR COALESCE(json_extract(execution_json, '$.clientToolCall.claimExpiresAt'), 0) <= ?
|
|
487
|
+
)
|
|
488
|
+
RETURNING *`,
|
|
489
|
+
claimToken,
|
|
490
|
+
claimExpiresAt,
|
|
491
|
+
now,
|
|
492
|
+
jobId,
|
|
493
|
+
userId,
|
|
494
|
+
callId,
|
|
495
|
+
now
|
|
496
|
+
);
|
|
497
|
+
return row ? toJob(row) : null;
|
|
498
|
+
}
|
|
499
|
+
async settleClientToolCall(jobId, userId, callId, claimToken, execution, toolCalls) {
|
|
500
|
+
const now = Date.now();
|
|
501
|
+
return await run(
|
|
502
|
+
`UPDATE background_jobs SET execution_json = ?, tool_calls_json = ?,
|
|
503
|
+
lease_owner = NULL, lease_expires_at = NULL, last_activity_at = ?
|
|
504
|
+
WHERE id = ? AND user_id = ? AND status = 'streaming'
|
|
505
|
+
AND json_extract(execution_json, '$.clientToolCall.callId') = ?
|
|
506
|
+
AND json_extract(execution_json, '$.clientToolCall.claimToken') = ?
|
|
507
|
+
AND COALESCE(json_extract(execution_json, '$.clientToolCall.claimExpiresAt'), 0) > ?`,
|
|
508
|
+
json(execution),
|
|
509
|
+
json(toolCalls),
|
|
510
|
+
now,
|
|
511
|
+
jobId,
|
|
512
|
+
userId,
|
|
513
|
+
callId,
|
|
514
|
+
claimToken,
|
|
515
|
+
now
|
|
516
|
+
) > 0;
|
|
517
|
+
}
|
|
458
518
|
async saveTerminalSnapshot(jobId, snapshot, leaseOwner) {
|
|
459
519
|
const values = [
|
|
460
520
|
snapshot.status,
|
|
@@ -528,7 +588,8 @@ export class SqliteBackgroundJobProvider {
|
|
|
528
588
|
}
|
|
529
589
|
async getActiveJobCount() {
|
|
530
590
|
const row = await one(
|
|
531
|
-
|
|
591
|
+
`SELECT COUNT(*) AS total FROM background_jobs WHERE status = 'streaming'
|
|
592
|
+
AND json_extract(execution_json, '$.clientToolCall') IS NULL`
|
|
532
593
|
);
|
|
533
594
|
return row?.total ?? 0;
|
|
534
595
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { PluginConnectionStore } from '~~/server/utils/plugins/connections/store/registry';
|
|
2
|
+
import { type SqliteRawDatabase } from '../db/kysely.js';
|
|
3
|
+
type SqliteDatabase = SqliteRawDatabase;
|
|
4
|
+
export declare function createSqlitePluginConnectionStore(options?: {
|
|
5
|
+
database?: SqliteDatabase;
|
|
6
|
+
}): PluginConnectionStore;
|
|
7
|
+
export {};
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getRawDb,
|
|
3
|
+
getSqliteDriver
|
|
4
|
+
} from "../db/kysely.js";
|
|
5
|
+
function parseStringArray(raw) {
|
|
6
|
+
try {
|
|
7
|
+
const parsed = JSON.parse(raw);
|
|
8
|
+
if (!Array.isArray(parsed)) return [];
|
|
9
|
+
return parsed.filter((value) => typeof value === "string");
|
|
10
|
+
} catch {
|
|
11
|
+
return [];
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function toConnection(row) {
|
|
15
|
+
return {
|
|
16
|
+
id: row.id,
|
|
17
|
+
ownerUserId: row.owner_user_id,
|
|
18
|
+
workspaceId: row.workspace_id,
|
|
19
|
+
pluginId: row.plugin_id,
|
|
20
|
+
providerId: row.provider_id,
|
|
21
|
+
...row.slot_id === null ? {} : { slotId: row.slot_id },
|
|
22
|
+
label: row.label,
|
|
23
|
+
scopes: parseStringArray(row.scopes),
|
|
24
|
+
revision: row.revision,
|
|
25
|
+
secretCiphertext: row.secret_ciphertext,
|
|
26
|
+
createdAt: row.created_at,
|
|
27
|
+
updatedAt: row.updated_at
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
function toEvidence(row) {
|
|
31
|
+
return {
|
|
32
|
+
connectionId: row.connection_id,
|
|
33
|
+
revision: row.revision,
|
|
34
|
+
operationId: row.operation_id,
|
|
35
|
+
ok: row.ok === 1,
|
|
36
|
+
...row.code === null ? {} : { code: row.code },
|
|
37
|
+
checkedAt: row.checked_at,
|
|
38
|
+
...row.detail === null ? {} : { detail: row.detail }
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
class SqlitePluginConnectionStore {
|
|
42
|
+
constructor(db) {
|
|
43
|
+
this.db = db;
|
|
44
|
+
this.#initialize();
|
|
45
|
+
}
|
|
46
|
+
#initialize() {
|
|
47
|
+
if (getSqliteDriver() !== "turso") {
|
|
48
|
+
this.db.pragma?.("journal_mode = WAL");
|
|
49
|
+
this.db.pragma?.("synchronous = NORMAL");
|
|
50
|
+
this.db.pragma?.("foreign_keys = ON");
|
|
51
|
+
}
|
|
52
|
+
this.db.exec?.(`
|
|
53
|
+
CREATE TABLE IF NOT EXISTS plugin_connections (
|
|
54
|
+
id TEXT PRIMARY KEY,
|
|
55
|
+
owner_user_id TEXT NOT NULL,
|
|
56
|
+
workspace_id TEXT NOT NULL,
|
|
57
|
+
plugin_id TEXT NOT NULL,
|
|
58
|
+
provider_id TEXT NOT NULL,
|
|
59
|
+
slot_id TEXT,
|
|
60
|
+
label TEXT NOT NULL DEFAULT '',
|
|
61
|
+
scopes TEXT NOT NULL DEFAULT '[]',
|
|
62
|
+
revision INTEGER NOT NULL DEFAULT 1,
|
|
63
|
+
secret_ciphertext TEXT NOT NULL,
|
|
64
|
+
created_at INTEGER NOT NULL,
|
|
65
|
+
updated_at INTEGER NOT NULL
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
CREATE TABLE IF NOT EXISTS plugin_connection_tests (
|
|
69
|
+
connection_id TEXT PRIMARY KEY,
|
|
70
|
+
revision INTEGER NOT NULL,
|
|
71
|
+
operation_id TEXT NOT NULL,
|
|
72
|
+
ok INTEGER NOT NULL,
|
|
73
|
+
code TEXT,
|
|
74
|
+
checked_at INTEGER NOT NULL,
|
|
75
|
+
detail TEXT,
|
|
76
|
+
FOREIGN KEY (connection_id) REFERENCES plugin_connections(id) ON DELETE CASCADE
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
CREATE INDEX IF NOT EXISTS idx_plugin_connections_workspace_plugin
|
|
80
|
+
ON plugin_connections (workspace_id, plugin_id);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_plugin_connections_owner
|
|
82
|
+
ON plugin_connections (owner_user_id);
|
|
83
|
+
`);
|
|
84
|
+
try {
|
|
85
|
+
this.db.prepare(`ALTER TABLE plugin_connections ADD COLUMN slot_id TEXT`).run();
|
|
86
|
+
} catch {
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Owner-scoped listing. Ownership is part of the SQL predicate so a workspace
|
|
91
|
+
* peer cannot discover another user's connections or their references.
|
|
92
|
+
*/
|
|
93
|
+
async list(input) {
|
|
94
|
+
const rows = input.pluginId === void 0 ? this.db.prepare(
|
|
95
|
+
`SELECT * FROM plugin_connections WHERE owner_user_id = ? AND workspace_id = ? ORDER BY created_at ASC`
|
|
96
|
+
).all(input.ownerUserId, input.workspaceId) : this.db.prepare(
|
|
97
|
+
`SELECT * FROM plugin_connections WHERE owner_user_id = ? AND workspace_id = ? AND plugin_id = ? ORDER BY created_at ASC`
|
|
98
|
+
).all(input.ownerUserId, input.workspaceId, input.pluginId);
|
|
99
|
+
return rows.map(toConnection);
|
|
100
|
+
}
|
|
101
|
+
async get(id) {
|
|
102
|
+
const row = this.db.prepare(`SELECT * FROM plugin_connections WHERE id = ?`).get(id);
|
|
103
|
+
return row ? toConnection(row) : null;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Insert-only creation. `DO NOTHING` is deliberate: an upsert here could
|
|
107
|
+
* reassign an existing record's owner, workspace, plugin or provider when two
|
|
108
|
+
* processes happen to generate the same id.
|
|
109
|
+
*/
|
|
110
|
+
async insert(connection) {
|
|
111
|
+
const result = this.db.prepare(
|
|
112
|
+
`INSERT INTO plugin_connections (
|
|
113
|
+
id, owner_user_id, workspace_id, plugin_id, provider_id, slot_id,
|
|
114
|
+
label, scopes, revision, secret_ciphertext, created_at, updated_at
|
|
115
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
116
|
+
ON CONFLICT(id) DO NOTHING`
|
|
117
|
+
).run(
|
|
118
|
+
connection.id,
|
|
119
|
+
connection.ownerUserId,
|
|
120
|
+
connection.workspaceId,
|
|
121
|
+
connection.pluginId,
|
|
122
|
+
connection.providerId,
|
|
123
|
+
connection.slotId ?? null,
|
|
124
|
+
connection.label,
|
|
125
|
+
JSON.stringify(connection.scopes),
|
|
126
|
+
connection.revision,
|
|
127
|
+
connection.secretCiphertext,
|
|
128
|
+
connection.createdAt,
|
|
129
|
+
connection.updatedAt
|
|
130
|
+
);
|
|
131
|
+
return result.changes > 0;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Compare-and-swap update. Identity columns are not in the SET list, so a
|
|
135
|
+
* retry cannot move the record; the revision predicate is the whole
|
|
136
|
+
* concurrency control.
|
|
137
|
+
*/
|
|
138
|
+
async update(update) {
|
|
139
|
+
const result = this.db.prepare(
|
|
140
|
+
`UPDATE plugin_connections
|
|
141
|
+
SET revision = ?,
|
|
142
|
+
secret_ciphertext = ?,
|
|
143
|
+
updated_at = ?,
|
|
144
|
+
scopes = COALESCE(?, scopes),
|
|
145
|
+
label = COALESCE(?, label)
|
|
146
|
+
WHERE id = ? AND revision = ?`
|
|
147
|
+
).run(
|
|
148
|
+
update.revision,
|
|
149
|
+
update.secretCiphertext,
|
|
150
|
+
update.updatedAt,
|
|
151
|
+
update.scopes === void 0 ? null : JSON.stringify(update.scopes),
|
|
152
|
+
update.label ?? null,
|
|
153
|
+
update.id,
|
|
154
|
+
update.expectedRevision
|
|
155
|
+
);
|
|
156
|
+
return result.changes > 0;
|
|
157
|
+
}
|
|
158
|
+
async delete(id) {
|
|
159
|
+
this.db.prepare(`DELETE FROM plugin_connections WHERE id = ?`).run(id);
|
|
160
|
+
this.db.prepare(`DELETE FROM plugin_connection_tests WHERE connection_id = ?`).run(id);
|
|
161
|
+
}
|
|
162
|
+
async getTestEvidence(connectionId) {
|
|
163
|
+
const row = this.db.prepare(`SELECT * FROM plugin_connection_tests WHERE connection_id = ?`).get(connectionId);
|
|
164
|
+
return row ? toEvidence(row) : null;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Stores evidence only when it belongs to the connection's current revision
|
|
168
|
+
* and is not older than what is already stored. A slow test that finishes
|
|
169
|
+
* after a credential rotation therefore cannot reinstate itself as current
|
|
170
|
+
* evidence, and an older completion cannot replace a newer result.
|
|
171
|
+
*/
|
|
172
|
+
async setTestEvidence(evidence) {
|
|
173
|
+
const result = this.db.prepare(
|
|
174
|
+
`INSERT INTO plugin_connection_tests (
|
|
175
|
+
connection_id, revision, operation_id, ok, code, checked_at, detail
|
|
176
|
+
)
|
|
177
|
+
SELECT ?, ?, ?, ?, ?, ?, ?
|
|
178
|
+
WHERE EXISTS (
|
|
179
|
+
SELECT 1 FROM plugin_connections
|
|
180
|
+
WHERE id = ? AND revision = ?
|
|
181
|
+
)
|
|
182
|
+
ON CONFLICT(connection_id) DO UPDATE SET
|
|
183
|
+
revision = excluded.revision,
|
|
184
|
+
operation_id = excluded.operation_id,
|
|
185
|
+
ok = excluded.ok,
|
|
186
|
+
code = excluded.code,
|
|
187
|
+
checked_at = excluded.checked_at,
|
|
188
|
+
detail = excluded.detail
|
|
189
|
+
WHERE excluded.revision > plugin_connection_tests.revision
|
|
190
|
+
OR (excluded.revision = plugin_connection_tests.revision
|
|
191
|
+
AND excluded.checked_at >= plugin_connection_tests.checked_at)`
|
|
192
|
+
).run(
|
|
193
|
+
evidence.connectionId,
|
|
194
|
+
evidence.revision,
|
|
195
|
+
evidence.operationId,
|
|
196
|
+
evidence.ok ? 1 : 0,
|
|
197
|
+
evidence.code ?? null,
|
|
198
|
+
evidence.checkedAt,
|
|
199
|
+
evidence.detail ?? null,
|
|
200
|
+
evidence.connectionId,
|
|
201
|
+
evidence.revision
|
|
202
|
+
);
|
|
203
|
+
return result.changes > 0;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
export function createSqlitePluginConnectionStore(options = {}) {
|
|
207
|
+
const db = options.database ?? getRawDb();
|
|
208
|
+
return new SqlitePluginConnectionStore(db);
|
|
209
|
+
}
|
|
@@ -5,6 +5,7 @@ import { registerAdminStoreProvider } from "~~/server/admin/stores/registry";
|
|
|
5
5
|
import { registerSyncGatewayAdapter } from "~~/server/sync/gateway/registry";
|
|
6
6
|
import { registerWebhookStore } from "~~/server/utils/webhooks/store/registry";
|
|
7
7
|
import { registerConnectStore } from "~~/server/connect/store/registry";
|
|
8
|
+
import { registerPluginConnectionStore } from "~~/server/utils/plugins/connections/store/registry";
|
|
8
9
|
import { registerRateLimitProvider } from "~~/server/utils/rate-limit/registry";
|
|
9
10
|
import { registerBackgroundJobProvider } from "~~/server/utils/background-jobs/registry";
|
|
10
11
|
import { createSqliteAuthWorkspaceStore } from "../auth/sqlite-auth-workspace-store.js";
|
|
@@ -17,6 +18,7 @@ import {
|
|
|
17
18
|
} from "../admin/stores/sqlite-store.js";
|
|
18
19
|
import { sqliteSyncAdminAdapter } from "../admin/adapters/sync-sqlite.js";
|
|
19
20
|
import { createSqliteConnectStore } from "../connect/sqlite-connect-store.js";
|
|
21
|
+
import { createSqlitePluginConnectionStore } from "../plugin-connections/sqlite-connection-store.js";
|
|
20
22
|
import { getSqliteDriver, initializeSqliteDb } from "../db/kysely.js";
|
|
21
23
|
import { runMigrations } from "../db/migrate.js";
|
|
22
24
|
import { sqliteRateLimitProvider } from "../rate-limit/sqlite-provider.js";
|
|
@@ -27,10 +29,12 @@ let d1RuntimeInitialization;
|
|
|
27
29
|
export default defineNitroPlugin(async (nitroApp) => {
|
|
28
30
|
const config = useRuntimeConfig();
|
|
29
31
|
if (!config.auth?.enabled) return;
|
|
32
|
+
const workspaceSelected = config.sync?.provider === SQLITE_PROVIDER_ID;
|
|
30
33
|
const syncSelected = config.sync?.enabled === true && config.sync?.provider === SQLITE_PROVIDER_ID;
|
|
31
34
|
const connectSelected = config.connect?.enabled === true && config.connect?.provider === SQLITE_PROVIDER_ID;
|
|
32
35
|
const backgroundJobsSelected = config.backgroundJobs?.enabled === true && config.backgroundJobs?.storageProvider === SQLITE_PROVIDER_ID;
|
|
33
|
-
if (!
|
|
36
|
+
if (!workspaceSelected && !connectSelected && !backgroundJobsSelected)
|
|
37
|
+
return;
|
|
34
38
|
const driver = getSqliteDriver();
|
|
35
39
|
if (driver === "d1" && connectSelected) {
|
|
36
40
|
throw new Error(
|
|
@@ -72,6 +76,17 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
72
76
|
"[or3-sqlite] Webhook persistence is unavailable with Cloudflare D1."
|
|
73
77
|
);
|
|
74
78
|
}
|
|
79
|
+
if (driver !== "d1") {
|
|
80
|
+
registerPluginConnectionStore({
|
|
81
|
+
id: SQLITE_PROVIDER_ID,
|
|
82
|
+
order: 100,
|
|
83
|
+
create: createSqlitePluginConnectionStore
|
|
84
|
+
});
|
|
85
|
+
} else {
|
|
86
|
+
console.warn(
|
|
87
|
+
"[or3-sqlite] Plugin connection persistence is unavailable with Cloudflare D1."
|
|
88
|
+
);
|
|
89
|
+
}
|
|
75
90
|
registerAdminStoreProvider({
|
|
76
91
|
id: SQLITE_PROVIDER_ID,
|
|
77
92
|
createWorkspaceAccessStore: createSqliteWorkspaceAccessStore,
|
|
@@ -369,6 +369,56 @@ function incomingWinsRevision(incoming, existing) {
|
|
|
369
369
|
opId: existing.opId || existing.op_id || ""
|
|
370
370
|
});
|
|
371
371
|
}
|
|
372
|
+
function materializedWinner(row, tombstone) {
|
|
373
|
+
if (tombstone && (!row || incomingWinsRevision(
|
|
374
|
+
{ clock: tombstone.clock, hlc: tombstone.hlc, opId: tombstone.op_id },
|
|
375
|
+
row
|
|
376
|
+
))) {
|
|
377
|
+
return {
|
|
378
|
+
kind: "delete",
|
|
379
|
+
revision: { clock: tombstone.clock, hlc: tombstone.hlc, opId: tombstone.op_id },
|
|
380
|
+
serverDeletedAt: tombstone.deleted_at
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
if (!row) return void 0;
|
|
384
|
+
const revision = { clock: row.clock, hlc: row.hlc, opId: row.op_id };
|
|
385
|
+
return row.deleted ? {
|
|
386
|
+
kind: "delete",
|
|
387
|
+
revision,
|
|
388
|
+
serverDeletedAt: tombstone?.op_id === row.op_id ? tombstone.deleted_at : row.updated_at
|
|
389
|
+
} : { kind: "put", payload: JSON.parse(row.data_json), revision };
|
|
390
|
+
}
|
|
391
|
+
function currentWinnerNative(raw, workspaceId, tableName, pk) {
|
|
392
|
+
const materializedTable = SYNCED_TABLE_MAP[tableName];
|
|
393
|
+
if (!materializedTable) return void 0;
|
|
394
|
+
const row = raw.prepare(
|
|
395
|
+
`SELECT data_json, deleted, clock, hlc, op_id, updated_at
|
|
396
|
+
FROM ${materializedTable} WHERE id = ? AND workspace_id = ?`
|
|
397
|
+
).get(pk, workspaceId);
|
|
398
|
+
const tombstone = raw.prepare(
|
|
399
|
+
`SELECT clock, hlc, op_id, deleted_at FROM tombstones
|
|
400
|
+
WHERE workspace_id = ? AND table_name = ? AND pk = ?`
|
|
401
|
+
).get(workspaceId, tableName, pk);
|
|
402
|
+
return materializedWinner(row, tombstone);
|
|
403
|
+
}
|
|
404
|
+
async function currentWinnerD1(workspaceId, tableName, pk) {
|
|
405
|
+
const materializedTable = SYNCED_TABLE_MAP[tableName];
|
|
406
|
+
if (!materializedTable) return void 0;
|
|
407
|
+
const [row] = await d1All(
|
|
408
|
+
`SELECT data_json, deleted, clock, hlc, op_id, updated_at
|
|
409
|
+
FROM "${materializedTable}" WHERE id = ? AND workspace_id = ?`,
|
|
410
|
+
pk,
|
|
411
|
+
workspaceId
|
|
412
|
+
);
|
|
413
|
+
const [tombstone] = await d1All(
|
|
414
|
+
`SELECT clock, hlc, op_id, deleted_at FROM tombstones
|
|
415
|
+
WHERE workspace_id = ? AND table_name = ? AND pk = ?`,
|
|
416
|
+
workspaceId,
|
|
417
|
+
tableName,
|
|
418
|
+
pk
|
|
419
|
+
);
|
|
420
|
+
return materializedWinner(row, tombstone);
|
|
421
|
+
}
|
|
372
422
|
export class SqliteSyncGatewayAdapter {
|
|
373
423
|
id = "sqlite";
|
|
374
424
|
get capabilities() {
|
|
@@ -1105,14 +1155,14 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1105
1155
|
});
|
|
1106
1156
|
continue;
|
|
1107
1157
|
}
|
|
1108
|
-
const
|
|
1109
|
-
`SELECT op_id FROM ${SYNCED_TABLE_MAP[op.tableName]} WHERE id = ? AND workspace_id = ?`
|
|
1110
|
-
).get(op.pk, workspaceId) : void 0;
|
|
1158
|
+
const winner = currentWinnerNative(raw, workspaceId, op.tableName, op.pk);
|
|
1111
1159
|
uniqueResults.push({
|
|
1112
1160
|
opId,
|
|
1113
1161
|
success: true,
|
|
1114
1162
|
serverVersion: existingLog.server_version,
|
|
1115
|
-
|
|
1163
|
+
replayed: true,
|
|
1164
|
+
applied: winner?.revision.opId === opId,
|
|
1165
|
+
...winner?.revision.opId === opId ? {} : { winner }
|
|
1116
1166
|
});
|
|
1117
1167
|
continue;
|
|
1118
1168
|
}
|
|
@@ -1240,7 +1290,10 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1240
1290
|
success: true,
|
|
1241
1291
|
serverVersion,
|
|
1242
1292
|
applied,
|
|
1243
|
-
...applied ? {} : {
|
|
1293
|
+
...applied ? {} : {
|
|
1294
|
+
payload: winnerPayload,
|
|
1295
|
+
winner: currentWinnerNative(raw, workspaceId, op.tableName, pkValue)
|
|
1296
|
+
}
|
|
1244
1297
|
});
|
|
1245
1298
|
} else if (op.operation === "delete") {
|
|
1246
1299
|
const existing = raw.prepare(
|
|
@@ -1316,7 +1369,10 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1316
1369
|
success: true,
|
|
1317
1370
|
serverVersion,
|
|
1318
1371
|
applied,
|
|
1319
|
-
...applied || !existing ? {} : { payload: JSON.parse(existing.data_json) }
|
|
1372
|
+
...applied || !existing ? {} : { payload: JSON.parse(existing.data_json) },
|
|
1373
|
+
...applied ? {} : {
|
|
1374
|
+
winner: currentWinnerNative(raw, workspaceId, op.tableName, pkValue)
|
|
1375
|
+
}
|
|
1320
1376
|
});
|
|
1321
1377
|
}
|
|
1322
1378
|
}
|
|
@@ -2431,21 +2487,15 @@ export class SqliteSyncGatewayAdapter {
|
|
|
2431
2487
|
if (rejected.has(op.stamp.opId)) continue;
|
|
2432
2488
|
const existing = existingOps.get(op.stamp.opId);
|
|
2433
2489
|
if (existing) {
|
|
2434
|
-
const
|
|
2435
|
-
|
|
2436
|
-
if (materializedTable) {
|
|
2437
|
-
const [row] = await d1All(
|
|
2438
|
-
`SELECT op_id FROM "${materializedTable}" WHERE id = ? AND workspace_id = ?`,
|
|
2439
|
-
op.pk,
|
|
2440
|
-
workspaceId
|
|
2441
|
-
);
|
|
2442
|
-
applied2 = row?.op_id === op.stamp.opId;
|
|
2443
|
-
}
|
|
2490
|
+
const winner = await currentWinnerD1(workspaceId, op.tableName, op.pk);
|
|
2491
|
+
const applied2 = winner?.revision.opId === op.stamp.opId;
|
|
2444
2492
|
const result2 = {
|
|
2445
2493
|
opId: op.stamp.opId,
|
|
2446
2494
|
success: true,
|
|
2447
2495
|
serverVersion: existing.server_version,
|
|
2448
|
-
|
|
2496
|
+
replayed: true,
|
|
2497
|
+
applied: applied2,
|
|
2498
|
+
...applied2 ? {} : { winner }
|
|
2449
2499
|
};
|
|
2450
2500
|
for (const index of indicesByOpId.get(op.stamp.opId) ?? []) {
|
|
2451
2501
|
resultSlots[index] = result2;
|
|
@@ -2478,7 +2528,10 @@ export class SqliteSyncGatewayAdapter {
|
|
|
2478
2528
|
success: true,
|
|
2479
2529
|
serverVersion,
|
|
2480
2530
|
applied,
|
|
2481
|
-
...applied ? {} : { payload: winnerPayload }
|
|
2531
|
+
...applied ? {} : { payload: winnerPayload },
|
|
2532
|
+
...applied ? {} : {
|
|
2533
|
+
winner: await currentWinnerD1(workspaceId, op.tableName, op.pk)
|
|
2534
|
+
}
|
|
2482
2535
|
};
|
|
2483
2536
|
for (const index of indicesByOpId.get(op.stamp.opId) ?? []) {
|
|
2484
2537
|
resultSlots[index] = result;
|