@context-use/open-sync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/package.json +44 -0
- package/src/api.ts +53 -0
- package/src/build.ts +6 -0
- package/src/connector/client.ts +156 -0
- package/src/connector/encryption-key.ts +23 -0
- package/src/connector/management.ts +76 -0
- package/src/connector/response.ts +35 -0
- package/src/db/client.ts +24 -0
- package/src/db/providers.sql +15 -0
- package/src/db/providers.sql.d.ts +2 -0
- package/src/db/providers.ts +24 -0
- package/src/db/schema.sql +45 -0
- package/src/db/schema.sql.d.ts +2 -0
- package/src/execution/diagnostics.ts +18 -0
- package/src/execution/provider.ts +37 -0
- package/src/execution/worker.ts +90 -0
- package/src/http/app.ts +21 -0
- package/src/http/controller.ts +95 -0
- package/src/http/index.ts +20 -0
- package/src/http/providers.ts +94 -0
- package/src/models/definition.ts +63 -0
- package/src/models/delivery-result.ts +22 -0
- package/src/models/delivery.ts +73 -0
- package/src/models/error.ts +11 -0
- package/src/models/identity.ts +11 -0
- package/src/models/installation.ts +37 -0
- package/src/models/json.ts +82 -0
- package/src/models/limits.ts +42 -0
- package/src/models/page.ts +69 -0
- package/src/models/provider-values.ts +42 -0
- package/src/models/providers.ts +34 -0
- package/src/models/registry.ts +74 -0
- package/src/models/validation.ts +24 -0
- package/src/open-sync.ts +182 -0
- package/src/repositories/acquisition/contract.ts +17 -0
- package/src/repositories/acquisition/lease.ts +118 -0
- package/src/repositories/acquisition/outbox.ts +52 -0
- package/src/repositories/acquisition/records.ts +37 -0
- package/src/repositories/acquisition/sqlite.ts +60 -0
- package/src/repositories/catalog/contract.ts +23 -0
- package/src/repositories/catalog/sqlite.ts +166 -0
- package/src/repositories/delivery/contract.ts +23 -0
- package/src/repositories/delivery/sqlite.ts +111 -0
- package/src/repositories/providers/contract.ts +15 -0
- package/src/repositories/providers/sqlite.ts +79 -0
- package/src/repositories/queue-usage.ts +21 -0
- package/src/repositories/rows.ts +50 -0
- package/src/runtime.ts +90 -0
- package/src/services/acquisition.ts +100 -0
- package/src/services/delivery.ts +53 -0
- package/src/services/management.ts +124 -0
- package/src/services/providers/service.ts +190 -0
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import { definitionKey, type SyncDefinition, type SyncPage } from '../../models/definition';
|
|
3
|
+
import type { DeliveredRecord } from '../../models/delivery';
|
|
4
|
+
import { fail } from '../../models/error';
|
|
5
|
+
import { canonicalJson } from '../../models/json';
|
|
6
|
+
import type { QueueLimits } from '../../models/limits';
|
|
7
|
+
import { preparePage } from '../../models/page';
|
|
8
|
+
import { hasQueueCapacity } from '../queue-usage';
|
|
9
|
+
import type { AcquisitionRepository, RunLease } from './contract';
|
|
10
|
+
import { assertRun, claimRun, finishRun } from './lease';
|
|
11
|
+
import { enqueue } from './outbox';
|
|
12
|
+
import { writeRecord } from './records';
|
|
13
|
+
|
|
14
|
+
export class SqliteAcquisition implements AcquisitionRepository {
|
|
15
|
+
constructor(
|
|
16
|
+
private readonly input: { db: Database; limits: QueueLimits; historyLimit: number },
|
|
17
|
+
) {}
|
|
18
|
+
claim(leaseMs: number) {
|
|
19
|
+
return claimRun({ ...this.input, leaseMs });
|
|
20
|
+
}
|
|
21
|
+
hasCapacity() {
|
|
22
|
+
return hasQueueCapacity(this.input);
|
|
23
|
+
}
|
|
24
|
+
commit(input: { lease: RunLease; page: SyncPage; definition: SyncDefinition }): void {
|
|
25
|
+
const { db, limits } = this.input;
|
|
26
|
+
const page = preparePage({ ...input, limits });
|
|
27
|
+
db.transaction(() => {
|
|
28
|
+
const installation = assertRun({ db, lease: input.lease });
|
|
29
|
+
if (definitionKey(input.definition) !== definitionKey(installation.definition)) {
|
|
30
|
+
fail('definition_conflict');
|
|
31
|
+
}
|
|
32
|
+
const records: DeliveredRecord[] = [];
|
|
33
|
+
for (const record of page.deliverable.records) {
|
|
34
|
+
const changed = writeRecord({ db, installation, record });
|
|
35
|
+
if (changed) {
|
|
36
|
+
records.push(changed);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
enqueue({ db, installation, records, limits });
|
|
40
|
+
db.query(
|
|
41
|
+
'UPDATE installations SET checkpoint=?,checkpoint_revision=checkpoint_revision+1 WHERE owner_id=? AND id=?',
|
|
42
|
+
).run(canonicalJson(page.checkpoint).json, installation.ownerId, installation.id);
|
|
43
|
+
db.query(
|
|
44
|
+
'UPDATE runs SET checkpoint_revision=checkpoint_revision+1,pages=pages+1 WHERE owner_id=? AND id=?',
|
|
45
|
+
).run(input.lease.ownerId, input.lease.id);
|
|
46
|
+
if (page.complete) {
|
|
47
|
+
finishRun({ db, lease: input.lease, state: 'succeeded', delay: installation.intervalMs });
|
|
48
|
+
}
|
|
49
|
+
}).immediate();
|
|
50
|
+
input.lease.checkpointRevision++;
|
|
51
|
+
}
|
|
52
|
+
finish(input: { lease: RunLease; state: string; delay: number }): void {
|
|
53
|
+
this.input.db
|
|
54
|
+
.transaction(() => {
|
|
55
|
+
assertRun({ db: this.input.db, lease: input.lease });
|
|
56
|
+
finishRun({ db: this.input.db, ...input });
|
|
57
|
+
})
|
|
58
|
+
.immediate();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { SyncDefinition } from '../../models/definition';
|
|
2
|
+
import type { Destination } from '../../models/delivery';
|
|
3
|
+
import type { Resource, Scope } from '../../models/identity';
|
|
4
|
+
import type { CreateInstallation, Installation, SyncRun } from '../../models/installation';
|
|
5
|
+
import type { JsonObject, JsonValue } from '../../models/json';
|
|
6
|
+
|
|
7
|
+
export interface CatalogRepository {
|
|
8
|
+
register(definition: SyncDefinition): void;
|
|
9
|
+
createDestination(
|
|
10
|
+
input: Scope & { type: string; version: string; config: JsonObject },
|
|
11
|
+
): Destination;
|
|
12
|
+
destinations(scope: Scope): Destination[];
|
|
13
|
+
createInstallation(input: CreateInstallation & { initialCheckpoint: JsonValue }): Installation;
|
|
14
|
+
installation(input: Resource): Installation;
|
|
15
|
+
installations(scope: Scope): Installation[];
|
|
16
|
+
runs(input: Resource & { offset: number }): {
|
|
17
|
+
runs: SyncRun[];
|
|
18
|
+
hasMore: boolean;
|
|
19
|
+
pageSize: number;
|
|
20
|
+
};
|
|
21
|
+
setEnabled(input: Resource & { enabled: boolean }): Installation;
|
|
22
|
+
queue(input: Resource & { checkpoint?: JsonValue }): void;
|
|
23
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type { SyncDefinition } from '../../models/definition';
|
|
3
|
+
import { fail } from '../../models/error';
|
|
4
|
+
import type { Resource, Scope } from '../../models/identity';
|
|
5
|
+
import type { CreateInstallation } from '../../models/installation';
|
|
6
|
+
import { canonicalJson, type JsonObject, type JsonValue } from '../../models/json';
|
|
7
|
+
import { defaultTiming } from '../../models/limits';
|
|
8
|
+
import { readDestination, readInstallation } from '../rows';
|
|
9
|
+
import type { CatalogRepository } from './contract';
|
|
10
|
+
|
|
11
|
+
export class SqliteCatalog implements CatalogRepository {
|
|
12
|
+
constructor(private readonly db: Database) {}
|
|
13
|
+
register(definition: SyncDefinition): void {
|
|
14
|
+
const manifest = canonicalJson(definition).json;
|
|
15
|
+
this.db
|
|
16
|
+
.transaction(() => {
|
|
17
|
+
const previous = this.db
|
|
18
|
+
.query<{ manifest: string }, [string, string]>(
|
|
19
|
+
'SELECT manifest FROM definitions WHERE id=? AND version=?',
|
|
20
|
+
)
|
|
21
|
+
.get(definition.id, definition.version);
|
|
22
|
+
if (previous && previous.manifest !== manifest) {
|
|
23
|
+
fail('definition_conflict');
|
|
24
|
+
}
|
|
25
|
+
this.db
|
|
26
|
+
.query('INSERT OR IGNORE INTO definitions VALUES (?,?,?,?)')
|
|
27
|
+
.run(definition.id, definition.version, definition.artifactId, manifest);
|
|
28
|
+
})
|
|
29
|
+
.immediate();
|
|
30
|
+
}
|
|
31
|
+
createDestination(input: Scope & { type: string; version: string; config: JsonObject }) {
|
|
32
|
+
const id = `dest_${crypto.randomUUID()}`;
|
|
33
|
+
this.db
|
|
34
|
+
.query('INSERT INTO destinations VALUES (?,?,?,?,?)')
|
|
35
|
+
.run(input.ownerId, id, input.type, input.version, canonicalJson(input.config).json);
|
|
36
|
+
return readDestination({ db: this.db, scope: { ...input, id } });
|
|
37
|
+
}
|
|
38
|
+
destinations(scope: Scope) {
|
|
39
|
+
return this.db
|
|
40
|
+
.query<{ id: string }, [string]>(
|
|
41
|
+
'SELECT id FROM destinations WHERE owner_id=? ORDER BY rowid',
|
|
42
|
+
)
|
|
43
|
+
.all(scope.ownerId)
|
|
44
|
+
.map(({ id }) => readDestination({ db: this.db, scope: { ...scope, id } }));
|
|
45
|
+
}
|
|
46
|
+
createInstallation(input: CreateInstallation & { initialCheckpoint: JsonValue }) {
|
|
47
|
+
const id = `sync_${crypto.randomUUID()}`;
|
|
48
|
+
this.db
|
|
49
|
+
.transaction(() => {
|
|
50
|
+
readDestination({ db: this.db, scope: { ...input, id: input.destinationId } });
|
|
51
|
+
this.db
|
|
52
|
+
.query(`INSERT INTO installations (owner_id,id,source_id,definition_id,definition_version,artifact_id,connection,config,destination_id,enabled,checkpoint,interval_ms,next_due_at,status)
|
|
53
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
54
|
+
.run(
|
|
55
|
+
input.ownerId,
|
|
56
|
+
id,
|
|
57
|
+
`source_${crypto.randomUUID()}`,
|
|
58
|
+
input.definition.id,
|
|
59
|
+
input.definition.version,
|
|
60
|
+
input.definition.artifactId,
|
|
61
|
+
input.connection ? canonicalJson(input.connection).json : null,
|
|
62
|
+
canonicalJson(input.config).json,
|
|
63
|
+
input.destinationId,
|
|
64
|
+
Number(input.enabled ?? true),
|
|
65
|
+
canonicalJson(input.initialCheckpoint).json,
|
|
66
|
+
input.intervalMs ?? defaultTiming.leaseMs,
|
|
67
|
+
Date.now(),
|
|
68
|
+
input.enabled === false ? 'disabled' : 'ready',
|
|
69
|
+
);
|
|
70
|
+
})
|
|
71
|
+
.immediate();
|
|
72
|
+
return this.installation({ ...input, id });
|
|
73
|
+
}
|
|
74
|
+
installation(input: Resource) {
|
|
75
|
+
return readInstallation({ db: this.db, scope: input });
|
|
76
|
+
}
|
|
77
|
+
installations(scope: Scope) {
|
|
78
|
+
return this.db
|
|
79
|
+
.query<{ id: string }, [string]>(
|
|
80
|
+
'SELECT id FROM installations WHERE owner_id=? ORDER BY rowid',
|
|
81
|
+
)
|
|
82
|
+
.all(scope.ownerId)
|
|
83
|
+
.map(({ id }) => this.installation({ ...scope, id }));
|
|
84
|
+
}
|
|
85
|
+
runs(input: Resource & { offset: number }) {
|
|
86
|
+
this.installation(input);
|
|
87
|
+
const limit = 50;
|
|
88
|
+
const rows = this.db
|
|
89
|
+
.query<
|
|
90
|
+
{
|
|
91
|
+
id: string;
|
|
92
|
+
state: string;
|
|
93
|
+
started_at: number;
|
|
94
|
+
completed_at: number | null;
|
|
95
|
+
pages: number;
|
|
96
|
+
checkpoint_revision: number;
|
|
97
|
+
},
|
|
98
|
+
[string, string, number, number]
|
|
99
|
+
>(
|
|
100
|
+
'SELECT id,state,started_at,completed_at,pages,checkpoint_revision FROM runs WHERE owner_id=? AND installation_id=? ORDER BY started_at DESC,rowid DESC LIMIT ? OFFSET ?',
|
|
101
|
+
)
|
|
102
|
+
.all(input.ownerId, input.id, limit + 1, input.offset);
|
|
103
|
+
return {
|
|
104
|
+
runs: rows.slice(0, limit).map((row) => ({
|
|
105
|
+
id: row.id,
|
|
106
|
+
state: row.state,
|
|
107
|
+
startedAt: row.started_at,
|
|
108
|
+
completedAt: row.completed_at,
|
|
109
|
+
pages: row.pages,
|
|
110
|
+
checkpointRevision: row.checkpoint_revision,
|
|
111
|
+
})),
|
|
112
|
+
hasMore: rows.length > limit,
|
|
113
|
+
pageSize: limit,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
setEnabled(input: Resource & { enabled: boolean }) {
|
|
117
|
+
return this.db
|
|
118
|
+
.transaction(() => {
|
|
119
|
+
this.installation(input);
|
|
120
|
+
this.db
|
|
121
|
+
.query(
|
|
122
|
+
'UPDATE installations SET enabled=?,binding_epoch=binding_epoch+1,status=?,next_due_at=? WHERE owner_id=? AND id=?',
|
|
123
|
+
)
|
|
124
|
+
.run(
|
|
125
|
+
Number(input.enabled),
|
|
126
|
+
input.enabled ? 'ready' : 'disabled',
|
|
127
|
+
Date.now(),
|
|
128
|
+
input.ownerId,
|
|
129
|
+
input.id,
|
|
130
|
+
);
|
|
131
|
+
this.db
|
|
132
|
+
.query(
|
|
133
|
+
"UPDATE runs SET state='cancelled',completed_at=? WHERE owner_id=? AND installation_id=? AND state='running'",
|
|
134
|
+
)
|
|
135
|
+
.run(Date.now(), input.ownerId, input.id);
|
|
136
|
+
return this.installation(input);
|
|
137
|
+
})
|
|
138
|
+
.immediate();
|
|
139
|
+
}
|
|
140
|
+
queue(input: Resource & { checkpoint?: JsonValue }): void {
|
|
141
|
+
this.db
|
|
142
|
+
.transaction(() => {
|
|
143
|
+
if (!this.installation(input).enabled) {
|
|
144
|
+
fail('disabled');
|
|
145
|
+
}
|
|
146
|
+
if (
|
|
147
|
+
this.db
|
|
148
|
+
.query("SELECT 1 FROM runs WHERE owner_id=? AND installation_id=? AND state='running'")
|
|
149
|
+
.get(input.ownerId, input.id)
|
|
150
|
+
) {
|
|
151
|
+
fail('busy');
|
|
152
|
+
}
|
|
153
|
+
if (input.checkpoint !== undefined) {
|
|
154
|
+
this.db
|
|
155
|
+
.query(
|
|
156
|
+
'UPDATE installations SET checkpoint=?,checkpoint_revision=checkpoint_revision+1 WHERE owner_id=? AND id=?',
|
|
157
|
+
)
|
|
158
|
+
.run(canonicalJson(input.checkpoint).json, input.ownerId, input.id);
|
|
159
|
+
}
|
|
160
|
+
this.db
|
|
161
|
+
.query("UPDATE installations SET next_due_at=?,status='ready' WHERE owner_id=? AND id=?")
|
|
162
|
+
.run(Date.now(), input.ownerId, input.id);
|
|
163
|
+
})
|
|
164
|
+
.immediate();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Delivery,
|
|
3
|
+
DeliveryPage,
|
|
4
|
+
DeliveryResult,
|
|
5
|
+
Destination,
|
|
6
|
+
QueueStatus,
|
|
7
|
+
} from '../../models/delivery';
|
|
8
|
+
import type { Resource, Scope } from '../../models/identity';
|
|
9
|
+
|
|
10
|
+
export interface DeliveryLease extends Scope {
|
|
11
|
+
delivery: Delivery;
|
|
12
|
+
destination: Destination;
|
|
13
|
+
workerId: string;
|
|
14
|
+
generation: number;
|
|
15
|
+
attempt: number;
|
|
16
|
+
}
|
|
17
|
+
export interface DeliveryRepository {
|
|
18
|
+
claim(leaseMs: number): DeliveryLease | undefined;
|
|
19
|
+
complete(input: { lease: DeliveryLease; result: DeliveryResult; delay: number }): void;
|
|
20
|
+
status(scope: Scope): QueueStatus;
|
|
21
|
+
pending(input: Scope & { offset: number }): DeliveryPage;
|
|
22
|
+
retry(input: Resource): void;
|
|
23
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type { DeliveryResult, PendingDelivery } from '../../models/delivery';
|
|
3
|
+
import { fail } from '../../models/error';
|
|
4
|
+
import { type Resource, type Scope, workerScope } from '../../models/identity';
|
|
5
|
+
import { queueUsage } from '../queue-usage';
|
|
6
|
+
import { type Row, readDestination } from '../rows';
|
|
7
|
+
import type { DeliveryLease, DeliveryRepository } from './contract';
|
|
8
|
+
|
|
9
|
+
export class SqliteDeliveries implements DeliveryRepository {
|
|
10
|
+
constructor(private readonly db: Database) {}
|
|
11
|
+
claim(leaseMs: number): DeliveryLease | undefined {
|
|
12
|
+
return this.db
|
|
13
|
+
.transaction(() => {
|
|
14
|
+
// A blocked delivery stops only its destination; ordering survives retries and restarts.
|
|
15
|
+
const row = this.db
|
|
16
|
+
.query<
|
|
17
|
+
Row,
|
|
18
|
+
[number, number]
|
|
19
|
+
>(`SELECT d.* FROM deliveries d WHERE d.state!='blocked' AND d.due_at<=?
|
|
20
|
+
AND (d.state='pending' OR d.expires_at<=?) AND NOT EXISTS (
|
|
21
|
+
SELECT 1 FROM deliveries prior WHERE prior.owner_id=d.owner_id AND prior.destination_id=d.destination_id AND prior.sequence<d.sequence)
|
|
22
|
+
ORDER BY d.sequence LIMIT 1`)
|
|
23
|
+
.get(Date.now(), Date.now());
|
|
24
|
+
if (!row) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const workerId = crypto.randomUUID();
|
|
28
|
+
const scope = workerScope(String(row.owner_id));
|
|
29
|
+
this.db
|
|
30
|
+
.query(
|
|
31
|
+
"UPDATE deliveries SET state='leased',worker_id=?,generation=generation+1,attempt=attempt+1,expires_at=? WHERE owner_id=? AND id=?",
|
|
32
|
+
)
|
|
33
|
+
.run(workerId, Date.now() + leaseMs, scope.ownerId, row.id!);
|
|
34
|
+
return {
|
|
35
|
+
...scope,
|
|
36
|
+
delivery: JSON.parse(String(row.body)),
|
|
37
|
+
destination: readDestination({
|
|
38
|
+
db: this.db,
|
|
39
|
+
scope: { ...scope, id: String(row.destination_id) },
|
|
40
|
+
}),
|
|
41
|
+
workerId,
|
|
42
|
+
generation: Number(row.generation) + 1,
|
|
43
|
+
attempt: Number(row.attempt) + 1,
|
|
44
|
+
};
|
|
45
|
+
})
|
|
46
|
+
.immediate();
|
|
47
|
+
}
|
|
48
|
+
complete(input: { lease: DeliveryLease; result: DeliveryResult; delay: number }): void {
|
|
49
|
+
const { lease, result } = input;
|
|
50
|
+
this.db
|
|
51
|
+
.transaction(() => {
|
|
52
|
+
if (
|
|
53
|
+
!this.db
|
|
54
|
+
.query(
|
|
55
|
+
"SELECT 1 FROM deliveries WHERE owner_id=? AND id=? AND state='leased' AND worker_id=? AND generation=? AND expires_at>?",
|
|
56
|
+
)
|
|
57
|
+
.get(lease.ownerId, lease.delivery.id, lease.workerId, lease.generation, Date.now())
|
|
58
|
+
) {
|
|
59
|
+
fail('lease_lost');
|
|
60
|
+
}
|
|
61
|
+
if (result.status === 'accepted') {
|
|
62
|
+
this.db
|
|
63
|
+
.query('DELETE FROM deliveries WHERE owner_id=? AND id=?')
|
|
64
|
+
.run(lease.ownerId, lease.delivery.id);
|
|
65
|
+
// Capacity is global; all paused acquisitions can compete for the released budget.
|
|
66
|
+
this.db
|
|
67
|
+
.query(
|
|
68
|
+
"UPDATE installations SET next_due_at=? WHERE enabled=1 AND status='waiting_for_capacity'",
|
|
69
|
+
)
|
|
70
|
+
.run(Date.now());
|
|
71
|
+
} else {
|
|
72
|
+
this.db
|
|
73
|
+
.query(
|
|
74
|
+
'UPDATE deliveries SET state=?,due_at=?,worker_id=NULL,expires_at=NULL,error_code=? WHERE owner_id=? AND id=?',
|
|
75
|
+
)
|
|
76
|
+
.run(
|
|
77
|
+
result.status === 'rejected' ? 'blocked' : 'pending',
|
|
78
|
+
Date.now() + input.delay,
|
|
79
|
+
result.code ?? null,
|
|
80
|
+
lease.ownerId,
|
|
81
|
+
lease.delivery.id,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
.immediate();
|
|
86
|
+
}
|
|
87
|
+
status(scope: Scope) {
|
|
88
|
+
return queueUsage({ db: this.db, ownerId: scope.ownerId });
|
|
89
|
+
}
|
|
90
|
+
pending(input: Scope & { offset: number }) {
|
|
91
|
+
const limit = 50;
|
|
92
|
+
const rows = this.db
|
|
93
|
+
.query<
|
|
94
|
+
PendingDelivery,
|
|
95
|
+
[string, number, number]
|
|
96
|
+
>(`SELECT id,installation_id AS installationId,destination_id AS destinationId,state,bytes,
|
|
97
|
+
record_count AS recordCount,attempt,due_at AS nextAttemptAt,error_code AS errorCode FROM deliveries WHERE owner_id=? ORDER BY sequence LIMIT ? OFFSET ?`)
|
|
98
|
+
.all(input.ownerId, limit + 1, input.offset);
|
|
99
|
+
return { deliveries: rows.slice(0, limit), hasMore: rows.length > limit, pageSize: limit };
|
|
100
|
+
}
|
|
101
|
+
retry(input: Resource): void {
|
|
102
|
+
const updated = this.db
|
|
103
|
+
.query(
|
|
104
|
+
"UPDATE deliveries SET state='pending',due_at=?,generation=generation+1,worker_id=NULL,expires_at=NULL,error_code=NULL WHERE owner_id=? AND id=?",
|
|
105
|
+
)
|
|
106
|
+
.run(Date.now(), input.ownerId, input.id);
|
|
107
|
+
if (!updated.changes) {
|
|
108
|
+
fail('not_found');
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ProviderAuthorization,
|
|
3
|
+
ProviderConnection,
|
|
4
|
+
ProviderScope,
|
|
5
|
+
} from '../../models/providers';
|
|
6
|
+
|
|
7
|
+
export interface ProviderRepository {
|
|
8
|
+
list(scope: ProviderScope): Promise<ProviderConnection[]>;
|
|
9
|
+
connection(input: ProviderScope & { id: string }): Promise<ProviderConnection | null>;
|
|
10
|
+
owns(input: ProviderScope & { connectorId: string }): Promise<boolean>;
|
|
11
|
+
start(input: ProviderScope & ProviderAuthorization): Promise<void>;
|
|
12
|
+
pending(input: ProviderScope & { id: string }): Promise<ProviderAuthorization | null>;
|
|
13
|
+
add(input: ProviderScope & ProviderConnection): Promise<void>;
|
|
14
|
+
complete(input: ProviderScope & ProviderConnection): Promise<void>;
|
|
15
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type {
|
|
3
|
+
ProviderAuthorization,
|
|
4
|
+
ProviderConnection,
|
|
5
|
+
ProviderScope,
|
|
6
|
+
} from '../../models/providers';
|
|
7
|
+
import type { ProviderRepository } from './contract';
|
|
8
|
+
|
|
9
|
+
// This database contains only Open Sync's ownership references, never provider credentials.
|
|
10
|
+
export class SqliteProviders implements ProviderRepository {
|
|
11
|
+
constructor(private readonly db: Database) {}
|
|
12
|
+
list(scope: ProviderScope) {
|
|
13
|
+
return Promise.resolve(
|
|
14
|
+
this.db
|
|
15
|
+
.query<ProviderConnection, [string]>(
|
|
16
|
+
'SELECT id, connector_id AS connectorId, account, service FROM provider_connections WHERE owner_id=? ORDER BY id',
|
|
17
|
+
)
|
|
18
|
+
.all(scope.ownerId),
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
connection(input: ProviderScope & { id: string }) {
|
|
22
|
+
return Promise.resolve(
|
|
23
|
+
this.db
|
|
24
|
+
.query<ProviderConnection, [string, string]>(
|
|
25
|
+
'SELECT id, connector_id AS connectorId, account, service FROM provider_connections WHERE owner_id=? AND id=?',
|
|
26
|
+
)
|
|
27
|
+
.get(input.ownerId, input.id),
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
owns(input: ProviderScope & { connectorId: string }) {
|
|
31
|
+
return Promise.resolve(
|
|
32
|
+
Boolean(
|
|
33
|
+
this.db
|
|
34
|
+
.query('SELECT id FROM provider_connections WHERE owner_id=? AND connector_id=?')
|
|
35
|
+
.get(input.ownerId, input.connectorId),
|
|
36
|
+
),
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
start(input: ProviderScope & ProviderAuthorization) {
|
|
40
|
+
this.db
|
|
41
|
+
.query(`INSERT INTO provider_authorizations(owner_id,id,request_id,service) VALUES (?,?,?,?)
|
|
42
|
+
ON CONFLICT(owner_id) DO UPDATE SET id=excluded.id,request_id=excluded.request_id,service=excluded.service`)
|
|
43
|
+
.run(input.ownerId, input.id, input.requestId, input.service);
|
|
44
|
+
return Promise.resolve();
|
|
45
|
+
}
|
|
46
|
+
pending(input: ProviderScope & { id: string }) {
|
|
47
|
+
return Promise.resolve(
|
|
48
|
+
this.db
|
|
49
|
+
.query<ProviderAuthorization, [string, string]>(
|
|
50
|
+
'SELECT id,request_id AS requestId,service FROM provider_authorizations WHERE owner_id=? AND id=?',
|
|
51
|
+
)
|
|
52
|
+
.get(input.ownerId, input.id),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
add(input: ProviderScope & ProviderConnection) {
|
|
56
|
+
this.db
|
|
57
|
+
.query(
|
|
58
|
+
'INSERT INTO provider_connections(owner_id,id,connector_id,account,service) VALUES (?,?,?,?,?)',
|
|
59
|
+
)
|
|
60
|
+
.run(input.ownerId, input.id, input.connectorId, input.account, input.service);
|
|
61
|
+
return Promise.resolve();
|
|
62
|
+
}
|
|
63
|
+
complete(input: ProviderScope & ProviderConnection) {
|
|
64
|
+
this.db
|
|
65
|
+
.transaction(() => {
|
|
66
|
+
// A superseded authorization must never claim a connection.
|
|
67
|
+
this.db
|
|
68
|
+
.query(`INSERT INTO provider_connections(owner_id,id,connector_id,account,service)
|
|
69
|
+
SELECT owner_id,id,?,?,? FROM provider_authorizations WHERE owner_id=? AND id=?
|
|
70
|
+
ON CONFLICT(owner_id,id) DO NOTHING`)
|
|
71
|
+
.run(input.connectorId, input.account, input.service, input.ownerId, input.id);
|
|
72
|
+
this.db
|
|
73
|
+
.query('DELETE FROM provider_authorizations WHERE owner_id=? AND id=?')
|
|
74
|
+
.run(input.ownerId, input.id);
|
|
75
|
+
})
|
|
76
|
+
.immediate();
|
|
77
|
+
return Promise.resolve();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type { QueueStatus } from '../models/delivery';
|
|
3
|
+
import type { QueueLimits } from '../models/limits';
|
|
4
|
+
export function queueUsage(input: { db: Database; ownerId?: string }): QueueStatus {
|
|
5
|
+
const row = input.db
|
|
6
|
+
.query<
|
|
7
|
+
QueueStatus,
|
|
8
|
+
[string | null, string | null]
|
|
9
|
+
>(`SELECT coalesce(sum(bytes),0) AS pendingBytes,
|
|
10
|
+
coalesce(sum(record_count),0) AS pendingRecords,count(*) AS pendingDeliveries,
|
|
11
|
+
coalesce(sum(state='blocked'),0) AS blockedDeliveries FROM deliveries WHERE (? IS NULL OR owner_id=?)`)
|
|
12
|
+
.get(input.ownerId ?? null, input.ownerId ?? null)!;
|
|
13
|
+
return row;
|
|
14
|
+
}
|
|
15
|
+
export function hasQueueCapacity(input: { db: Database; limits: QueueLimits }): boolean {
|
|
16
|
+
const usage = queueUsage({ db: input.db });
|
|
17
|
+
return (
|
|
18
|
+
usage.pendingBytes < input.limits.maxPendingBytes &&
|
|
19
|
+
usage.pendingRecords < input.limits.maxPendingRecords
|
|
20
|
+
);
|
|
21
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { Database } from 'bun:sqlite';
|
|
2
|
+
import type { Destination } from '../models/delivery';
|
|
3
|
+
import { fail } from '../models/error';
|
|
4
|
+
import type { Resource } from '../models/identity';
|
|
5
|
+
import type { Installation } from '../models/installation';
|
|
6
|
+
|
|
7
|
+
export type Row = Record<string, string | number | null>;
|
|
8
|
+
export function readInstallation(input: { db: Database; scope: Resource }): Installation {
|
|
9
|
+
const row = input.db
|
|
10
|
+
.query<Row, [string, string]>('SELECT * FROM installations WHERE owner_id=? AND id=?')
|
|
11
|
+
.get(input.scope.ownerId, input.scope.id);
|
|
12
|
+
if (!row) {
|
|
13
|
+
fail('not_found');
|
|
14
|
+
}
|
|
15
|
+
return {
|
|
16
|
+
id: String(row.id),
|
|
17
|
+
ownerId: String(row.owner_id),
|
|
18
|
+
sourceId: String(row.source_id),
|
|
19
|
+
definition: {
|
|
20
|
+
id: String(row.definition_id),
|
|
21
|
+
version: String(row.definition_version),
|
|
22
|
+
artifactId: String(row.artifact_id),
|
|
23
|
+
},
|
|
24
|
+
connection: row.connection === null ? undefined : JSON.parse(String(row.connection)),
|
|
25
|
+
config: JSON.parse(String(row.config)),
|
|
26
|
+
destinationId: String(row.destination_id),
|
|
27
|
+
enabled: row.enabled === 1,
|
|
28
|
+
bindingEpoch: Number(row.binding_epoch),
|
|
29
|
+
checkpoint: JSON.parse(String(row.checkpoint)),
|
|
30
|
+
checkpointRevision: Number(row.checkpoint_revision),
|
|
31
|
+
intervalMs: Number(row.interval_ms),
|
|
32
|
+
nextDueAt: Number(row.next_due_at),
|
|
33
|
+
status: String(row.status),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
export function readDestination(input: { db: Database; scope: Resource }): Destination {
|
|
37
|
+
const row = input.db
|
|
38
|
+
.query<Row, [string, string]>('SELECT * FROM destinations WHERE owner_id=? AND id=?')
|
|
39
|
+
.get(input.scope.ownerId, input.scope.id);
|
|
40
|
+
if (!row) {
|
|
41
|
+
fail('not_found');
|
|
42
|
+
}
|
|
43
|
+
return {
|
|
44
|
+
id: String(row.id),
|
|
45
|
+
ownerId: String(row.owner_id),
|
|
46
|
+
type: String(row.type),
|
|
47
|
+
version: String(row.version),
|
|
48
|
+
config: JSON.parse(String(row.config)),
|
|
49
|
+
};
|
|
50
|
+
}
|
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { openDatabase } from './db/client';
|
|
2
|
+
import { type Logger, safeLogger } from './execution/diagnostics';
|
|
3
|
+
import type { ProviderGateway } from './execution/provider';
|
|
4
|
+
import { Worker } from './execution/worker';
|
|
5
|
+
import type { SyncRegistration } from './models/definition';
|
|
6
|
+
import type { DestinationType } from './models/delivery';
|
|
7
|
+
import { fail } from './models/error';
|
|
8
|
+
import {
|
|
9
|
+
defaultLimits,
|
|
10
|
+
defaultTiming,
|
|
11
|
+
positive,
|
|
12
|
+
type QueueLimits,
|
|
13
|
+
type Timing,
|
|
14
|
+
} from './models/limits';
|
|
15
|
+
import { Registry } from './models/registry';
|
|
16
|
+
import { SqliteAcquisition } from './repositories/acquisition/sqlite';
|
|
17
|
+
import { SqliteCatalog } from './repositories/catalog/sqlite';
|
|
18
|
+
import { SqliteDeliveries } from './repositories/delivery/sqlite';
|
|
19
|
+
import { AcquisitionService } from './services/acquisition';
|
|
20
|
+
import { DeliveryService } from './services/delivery';
|
|
21
|
+
import { SyncManagement } from './services/management';
|
|
22
|
+
|
|
23
|
+
export interface SyncRuntimeOptions {
|
|
24
|
+
databasePath: string;
|
|
25
|
+
definitions: readonly SyncRegistration[];
|
|
26
|
+
destinationTypes: Readonly<Record<string, DestinationType>>;
|
|
27
|
+
connector?: ProviderGateway;
|
|
28
|
+
limits?: Partial<QueueLimits>;
|
|
29
|
+
timing?: Partial<Timing>;
|
|
30
|
+
onEvent?: Logger;
|
|
31
|
+
}
|
|
32
|
+
/** Bun headless composition root. The host owns listeners, authentication and process lifecycle. */
|
|
33
|
+
export function createSyncRuntime(options: SyncRuntimeOptions) {
|
|
34
|
+
const timing = { ...defaultTiming, ...options.timing };
|
|
35
|
+
const limits = { ...defaultLimits, ...options.limits };
|
|
36
|
+
for (const value of [...Object.values(timing), ...Object.values(limits)]) {
|
|
37
|
+
positive(value);
|
|
38
|
+
}
|
|
39
|
+
if (timing.timeoutMs >= timing.leaseMs) {
|
|
40
|
+
fail('timeout_must_be_shorter_than_lease');
|
|
41
|
+
}
|
|
42
|
+
const registry = new Registry({
|
|
43
|
+
definitions: options.definitions,
|
|
44
|
+
destinations: options.destinationTypes,
|
|
45
|
+
});
|
|
46
|
+
const db = openDatabase(options.databasePath);
|
|
47
|
+
try {
|
|
48
|
+
const catalog = new SqliteCatalog(db);
|
|
49
|
+
for (const definition of registry.definitions()) {
|
|
50
|
+
catalog.register(definition);
|
|
51
|
+
}
|
|
52
|
+
const deliveries = new SqliteDeliveries(db);
|
|
53
|
+
const log = safeLogger(options.onEvent);
|
|
54
|
+
const worker = new Worker({
|
|
55
|
+
acquisition: new AcquisitionService({
|
|
56
|
+
repository: new SqliteAcquisition({ db, limits, historyLimit: timing.historyLimit }),
|
|
57
|
+
registry,
|
|
58
|
+
gateway: options.connector,
|
|
59
|
+
timing,
|
|
60
|
+
log,
|
|
61
|
+
}),
|
|
62
|
+
delivery: new DeliveryService({ repository: deliveries, registry, timing }),
|
|
63
|
+
timing,
|
|
64
|
+
log,
|
|
65
|
+
});
|
|
66
|
+
const api = new SyncManagement({
|
|
67
|
+
catalog,
|
|
68
|
+
deliveries,
|
|
69
|
+
registry,
|
|
70
|
+
worker,
|
|
71
|
+
gateway: options.connector,
|
|
72
|
+
limits,
|
|
73
|
+
timeoutMs: timing.timeoutMs,
|
|
74
|
+
});
|
|
75
|
+
let closing: Promise<void> | undefined;
|
|
76
|
+
return {
|
|
77
|
+
api,
|
|
78
|
+
start: () => worker.start(),
|
|
79
|
+
tick: () => worker.tick(),
|
|
80
|
+
close: () => {
|
|
81
|
+
closing ??= worker.close().finally(() => db.close());
|
|
82
|
+
return closing;
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
} catch (error) {
|
|
86
|
+
db.close();
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
export type SyncRuntime = ReturnType<typeof createSyncRuntime>;
|