@context-use/open-sync 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -0
- package/package.json +7 -4
- package/src/api.ts +18 -14
- package/src/build.ts +3 -21
- package/src/connector/client.ts +61 -28
- package/src/connector/failure.ts +49 -0
- package/src/connector/file.ts +104 -0
- package/src/connector/response.ts +22 -1
- package/src/db/client.ts +1 -11
- package/src/db/providers.sql +2 -3
- package/src/db/providers.ts +1 -11
- package/src/db/schema.sql +41 -36
- package/src/execution/abortable.ts +12 -0
- package/src/execution/diagnostics.ts +1 -2
- package/src/execution/provider.ts +1 -0
- package/src/execution/worker.ts +162 -39
- package/src/http/controller.ts +63 -42
- package/src/http/providers.ts +13 -0
- package/src/models/asset.ts +61 -0
- package/src/models/definition.ts +24 -15
- package/src/models/delivery.ts +25 -31
- package/src/models/error.ts +12 -1
- package/src/models/limits.ts +8 -3
- package/src/models/metadata.ts +42 -0
- package/src/models/page.ts +68 -13
- package/src/models/provider-catalog.ts +57 -0
- package/src/models/provider-path.ts +25 -0
- package/src/models/providers.ts +13 -2
- package/src/models/record.ts +22 -0
- package/src/models/registry.ts +15 -15
- package/src/models/source-http-error.ts +37 -0
- package/src/models/sync.ts +71 -0
- package/src/models/validation.ts +1 -1
- package/src/open-sync.ts +31 -7
- package/src/repositories/acquisition/contract.ts +18 -10
- package/src/repositories/acquisition/lease.ts +80 -94
- package/src/repositories/acquisition/outbox.ts +22 -22
- package/src/repositories/acquisition/records.ts +44 -13
- package/src/repositories/acquisition/sqlite.ts +78 -31
- package/src/repositories/assets/capacity.ts +16 -0
- package/src/repositories/assets/contract.ts +25 -0
- package/src/repositories/assets/filesystem.ts +103 -0
- package/src/repositories/assets/sqlite.ts +174 -0
- package/src/repositories/catalog/contract.ts +16 -18
- package/src/repositories/catalog/sqlite.ts +107 -115
- package/src/repositories/delivery/contract.ts +7 -11
- package/src/repositories/delivery/lease.ts +16 -0
- package/src/repositories/delivery/sqlite.ts +57 -34
- package/src/repositories/providers/contract.ts +2 -1
- package/src/repositories/providers/sqlite.ts +22 -6
- package/src/repositories/queue-usage.ts +7 -8
- package/src/repositories/rows.ts +10 -29
- package/src/runtime.ts +27 -6
- package/src/services/acquisition.ts +178 -52
- package/src/services/delivery.ts +39 -19
- package/src/services/management.ts +108 -44
- package/src/services/providers/service.ts +187 -31
- package/src/build/connector.ts +0 -143
- package/src/models/installation.ts +0 -37
package/src/build/connector.ts
DELETED
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
import { access, cp, mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import type { BunPlugin } from 'bun';
|
|
4
|
-
|
|
5
|
-
// This adapter deliberately supports one published layout. Review it when upgrading Connector.
|
|
6
|
-
const supportedVersion = '1.6.0';
|
|
7
|
-
const indexVersion = 1;
|
|
8
|
-
interface CatalogEntry {
|
|
9
|
-
file: string;
|
|
10
|
-
bytes: number;
|
|
11
|
-
provider: { service: string };
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
function unsupported(detail: string): never {
|
|
15
|
-
throw new Error(
|
|
16
|
-
`Unsupported Open Connector package: ${detail}. Review the Open Sync build adapter.`,
|
|
17
|
-
);
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/** Parse only the pinned generator's format; never evaluate dependency code during preparation. */
|
|
21
|
-
function executorRegistry(source: string): Map<string, string> {
|
|
22
|
-
const lines = source.trim().split('\n');
|
|
23
|
-
if (
|
|
24
|
-
lines.shift() !== '/** Generated lazy imports for provider executors. Do not hand-edit. */' ||
|
|
25
|
-
lines.shift() !== 'export const executorModules = {' ||
|
|
26
|
-
lines.pop() !== '};'
|
|
27
|
-
) {
|
|
28
|
-
unsupported('executor registry format changed');
|
|
29
|
-
}
|
|
30
|
-
const modules = new Map<string, string>();
|
|
31
|
-
for (const line of lines) {
|
|
32
|
-
const match =
|
|
33
|
-
/^ {4}(?:"([\w-]+)"|([\w]+)): \(\) => import\("(\.\/[\w-]+\/executors\.js)"\),$/.exec(line);
|
|
34
|
-
const service = match?.[1] ?? match?.[2];
|
|
35
|
-
const path = match?.[3];
|
|
36
|
-
if (!service || path !== `./${service}/executors.js` || modules.has(service)) {
|
|
37
|
-
unsupported('executor registry entry changed');
|
|
38
|
-
}
|
|
39
|
-
modules.set(service, path);
|
|
40
|
-
}
|
|
41
|
-
return modules;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function catalogEntries(value: unknown): CatalogEntry[] {
|
|
45
|
-
const index = value as { version?: unknown; providers?: CatalogEntry[] } | null;
|
|
46
|
-
if (index?.version !== indexVersion || !Array.isArray(index.providers)) {
|
|
47
|
-
unsupported('catalog index format changed');
|
|
48
|
-
}
|
|
49
|
-
const services = new Set<string>();
|
|
50
|
-
for (const entry of index.providers) {
|
|
51
|
-
const service = entry?.provider?.service;
|
|
52
|
-
if (
|
|
53
|
-
typeof service !== 'string' ||
|
|
54
|
-
!/^[\w-]+$/.test(service) ||
|
|
55
|
-
entry.file !== `${service}.json` ||
|
|
56
|
-
!Number.isInteger(entry.bytes) ||
|
|
57
|
-
entry.bytes < 0 ||
|
|
58
|
-
services.has(service)
|
|
59
|
-
) {
|
|
60
|
-
unsupported('catalog index entry changed');
|
|
61
|
-
}
|
|
62
|
-
services.add(service);
|
|
63
|
-
}
|
|
64
|
-
return index.providers;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/** All knowledge of Connector's private package layout stays behind this build adapter. */
|
|
68
|
-
export async function prepareConnector(input: {
|
|
69
|
-
root: string;
|
|
70
|
-
providers: readonly string[];
|
|
71
|
-
assets: string;
|
|
72
|
-
}): Promise<BunPlugin> {
|
|
73
|
-
const root = await realpath(input.root);
|
|
74
|
-
const metadata = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
|
|
75
|
-
if (metadata.name !== '@oomol-lab/open-connector' || metadata.version !== supportedVersion) {
|
|
76
|
-
unsupported(
|
|
77
|
-
`expected @oomol-lab/open-connector ${supportedVersion}, found ${metadata.version}`,
|
|
78
|
-
);
|
|
79
|
-
}
|
|
80
|
-
const registry = join(root, 'src/providers/registry.generated.js');
|
|
81
|
-
const modules = executorRegistry(await readFile(registry, 'utf8'));
|
|
82
|
-
const sourceAssets = join(root, 'assets/open-connector');
|
|
83
|
-
const entries = catalogEntries(
|
|
84
|
-
JSON.parse(await readFile(join(sourceAssets, 'catalog/apps-index.json'), 'utf8')),
|
|
85
|
-
);
|
|
86
|
-
const selected = input.providers.map((service) => {
|
|
87
|
-
const entry = entries.find((item) => item.provider.service === service);
|
|
88
|
-
if (!entry || !modules.has(service)) {
|
|
89
|
-
throw new Error(
|
|
90
|
-
`Cannot bundle Open Connector provider ${JSON.stringify(service)}: catalog or executor is missing.`,
|
|
91
|
-
);
|
|
92
|
-
}
|
|
93
|
-
return entry;
|
|
94
|
-
});
|
|
95
|
-
await mkdir(join(input.assets, 'catalog/apps'), { recursive: true });
|
|
96
|
-
for (const entry of selected) {
|
|
97
|
-
await access(join(root, 'src/providers', modules.get(entry.provider.service)!));
|
|
98
|
-
const content = await readFile(join(sourceAssets, 'catalog/apps', entry.file));
|
|
99
|
-
if (
|
|
100
|
-
content.byteLength !== entry.bytes ||
|
|
101
|
-
JSON.parse(content.toString()).service !== entry.provider.service
|
|
102
|
-
) {
|
|
103
|
-
unsupported(`catalog file does not match its index: ${entry.file}`);
|
|
104
|
-
}
|
|
105
|
-
await writeFile(join(input.assets, 'catalog/apps', entry.file), content);
|
|
106
|
-
}
|
|
107
|
-
if (!selected.length) {
|
|
108
|
-
// Bun embeds files, so retain the directory that Connector enumerates even with zero providers.
|
|
109
|
-
await writeFile(join(input.assets, 'catalog/apps/empty'), '');
|
|
110
|
-
}
|
|
111
|
-
await writeFile(
|
|
112
|
-
join(input.assets, 'catalog/apps-index.json'),
|
|
113
|
-
JSON.stringify({ version: indexVersion, providers: selected }),
|
|
114
|
-
);
|
|
115
|
-
await cp(join(sourceAssets, 'migrations'), join(input.assets, 'migrations'), { recursive: true });
|
|
116
|
-
const contents = `export const executorModules = {\n${selected
|
|
117
|
-
.map(
|
|
118
|
-
({ provider }) =>
|
|
119
|
-
`${JSON.stringify(provider.service)}: () => import(${JSON.stringify(modules.get(provider.service))}),`,
|
|
120
|
-
)
|
|
121
|
-
.join('\n')}\n};`;
|
|
122
|
-
return {
|
|
123
|
-
name: 'open-sync-providers',
|
|
124
|
-
setup(build) {
|
|
125
|
-
let replaced = false;
|
|
126
|
-
build.onStart(() => {
|
|
127
|
-
replaced = false;
|
|
128
|
-
});
|
|
129
|
-
build.onLoad({ filter: /[/\\]providers[/\\]registry\.generated\.js$/ }, ({ path }) => {
|
|
130
|
-
if (path !== registry) {
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
replaced = true;
|
|
134
|
-
return { contents, loader: 'js' };
|
|
135
|
-
});
|
|
136
|
-
build.onEnd((result) => {
|
|
137
|
-
if (result.success && !replaced) {
|
|
138
|
-
unsupported('the build did not load the expected executor registry');
|
|
139
|
-
}
|
|
140
|
-
});
|
|
141
|
-
},
|
|
142
|
-
};
|
|
143
|
-
}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import type { ConnectionRef, DefinitionRef } from './definition';
|
|
2
|
-
import type { Scope } from './identity';
|
|
3
|
-
import type { JsonObject, JsonValue } from './json';
|
|
4
|
-
|
|
5
|
-
export interface Installation {
|
|
6
|
-
id: string;
|
|
7
|
-
ownerId: string;
|
|
8
|
-
sourceId: string;
|
|
9
|
-
definition: DefinitionRef;
|
|
10
|
-
connection?: ConnectionRef;
|
|
11
|
-
config: JsonObject;
|
|
12
|
-
destinationId: string;
|
|
13
|
-
enabled: boolean;
|
|
14
|
-
bindingEpoch: number;
|
|
15
|
-
checkpoint: JsonValue;
|
|
16
|
-
checkpointRevision: number;
|
|
17
|
-
intervalMs: number;
|
|
18
|
-
nextDueAt: number;
|
|
19
|
-
status: string;
|
|
20
|
-
}
|
|
21
|
-
export interface CreateInstallation extends Scope {
|
|
22
|
-
definition: DefinitionRef;
|
|
23
|
-
connection?: ConnectionRef;
|
|
24
|
-
config: JsonObject;
|
|
25
|
-
destinationId: string;
|
|
26
|
-
intervalMs?: number;
|
|
27
|
-
enabled?: boolean;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export interface SyncRun {
|
|
31
|
-
id: string;
|
|
32
|
-
state: string;
|
|
33
|
-
startedAt: number;
|
|
34
|
-
completedAt: number | null;
|
|
35
|
-
pages: number;
|
|
36
|
-
checkpointRevision: number;
|
|
37
|
-
}
|