@context-use/open-sync 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -0
- package/package.json +7 -4
- package/src/api.ts +18 -14
- package/src/build.ts +3 -21
- package/src/connector/client.ts +61 -28
- package/src/connector/failure.ts +49 -0
- package/src/connector/file.ts +104 -0
- package/src/connector/response.ts +22 -1
- package/src/db/client.ts +1 -11
- package/src/db/providers.sql +2 -3
- package/src/db/providers.ts +1 -11
- package/src/db/schema.sql +41 -36
- package/src/execution/abortable.ts +12 -0
- package/src/execution/diagnostics.ts +1 -2
- package/src/execution/provider.ts +1 -0
- package/src/execution/worker.ts +162 -39
- package/src/http/controller.ts +63 -42
- package/src/http/providers.ts +13 -0
- package/src/models/asset.ts +61 -0
- package/src/models/definition.ts +24 -15
- package/src/models/delivery.ts +25 -31
- package/src/models/error.ts +12 -1
- package/src/models/limits.ts +8 -3
- package/src/models/metadata.ts +42 -0
- package/src/models/page.ts +68 -13
- package/src/models/provider-catalog.ts +57 -0
- package/src/models/provider-path.ts +25 -0
- package/src/models/providers.ts +13 -2
- package/src/models/record.ts +22 -0
- package/src/models/registry.ts +15 -15
- package/src/models/source-http-error.ts +37 -0
- package/src/models/sync.ts +71 -0
- package/src/models/validation.ts +1 -1
- package/src/open-sync.ts +31 -7
- package/src/repositories/acquisition/contract.ts +18 -10
- package/src/repositories/acquisition/lease.ts +80 -94
- package/src/repositories/acquisition/outbox.ts +22 -22
- package/src/repositories/acquisition/records.ts +44 -13
- package/src/repositories/acquisition/sqlite.ts +78 -31
- package/src/repositories/assets/capacity.ts +16 -0
- package/src/repositories/assets/contract.ts +25 -0
- package/src/repositories/assets/filesystem.ts +103 -0
- package/src/repositories/assets/sqlite.ts +174 -0
- package/src/repositories/catalog/contract.ts +16 -18
- package/src/repositories/catalog/sqlite.ts +107 -115
- package/src/repositories/delivery/contract.ts +7 -11
- package/src/repositories/delivery/lease.ts +16 -0
- package/src/repositories/delivery/sqlite.ts +57 -34
- package/src/repositories/providers/contract.ts +2 -1
- package/src/repositories/providers/sqlite.ts +22 -6
- package/src/repositories/queue-usage.ts +7 -8
- package/src/repositories/rows.ts +10 -29
- package/src/runtime.ts +27 -6
- package/src/services/acquisition.ts +178 -52
- package/src/services/delivery.ts +39 -19
- package/src/services/management.ts +108 -44
- package/src/services/providers/service.ts +187 -31
- package/src/build/connector.ts +0 -143
- package/src/models/installation.ts +0 -37
package/src/models/limits.ts
CHANGED
|
@@ -4,20 +4,25 @@ export interface QueueLimits {
|
|
|
4
4
|
maxPendingRecords: number;
|
|
5
5
|
maxPageBytes: number;
|
|
6
6
|
maxPageRecords: number;
|
|
7
|
+
maxPageAssets: number;
|
|
8
|
+
maxAssetBytes: number;
|
|
9
|
+
maxPendingAssetBytes: number;
|
|
7
10
|
}
|
|
8
11
|
export const defaultLimits: QueueLimits = {
|
|
9
12
|
maxPendingBytes: 67_108_864,
|
|
10
13
|
maxPendingRecords: 100_000,
|
|
11
14
|
maxPageBytes: 1_048_576,
|
|
12
15
|
maxPageRecords: 1000,
|
|
16
|
+
maxPageAssets: 1000,
|
|
17
|
+
maxAssetBytes: 104_857_600,
|
|
18
|
+
maxPendingAssetBytes: 1_073_741_824,
|
|
13
19
|
};
|
|
14
20
|
export const defaultTiming = {
|
|
15
|
-
pollMs: 1000,
|
|
16
21
|
leaseMs: 60_000,
|
|
17
22
|
timeoutMs: 30_000,
|
|
18
23
|
retryMs: 30_000,
|
|
19
|
-
|
|
20
|
-
|
|
24
|
+
sourceConcurrency: 4,
|
|
25
|
+
deliveryConcurrency: 4,
|
|
21
26
|
};
|
|
22
27
|
export type Timing = typeof defaultTiming;
|
|
23
28
|
export function positive(value: number): number {
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { fail } from './error';
|
|
2
|
+
import { validate } from './validation';
|
|
3
|
+
|
|
4
|
+
export interface SourceTimestamps {
|
|
5
|
+
/** Creation time in the source, when known. ISO 8601; delivered as UTC. */
|
|
6
|
+
createdAt?: string;
|
|
7
|
+
/** Last modification time in the source, when known. Never the sync time. */
|
|
8
|
+
updatedAt?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface RecordMetadata extends SourceTimestamps {
|
|
12
|
+
/** Plain text for listings; acquisition collapses whitespace and keeps at most 200 characters. */
|
|
13
|
+
preview?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function normalizeSourceTimestamps(input: SourceTimestamps): SourceTimestamps {
|
|
17
|
+
const result: SourceTimestamps = {};
|
|
18
|
+
for (const field of ['createdAt', 'updatedAt'] as const) {
|
|
19
|
+
if (input[field] !== undefined) {
|
|
20
|
+
validate({ value: input[field], schema: { type: 'string', format: 'date-time' } });
|
|
21
|
+
result[field] = new Date(input[field]).toISOString();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function normalizeRecordMetadata(input: RecordMetadata): RecordMetadata {
|
|
28
|
+
const timestamps = normalizeSourceTimestamps(input);
|
|
29
|
+
if (input.preview === undefined) {
|
|
30
|
+
return timestamps;
|
|
31
|
+
}
|
|
32
|
+
if (typeof input.preview !== 'string') {
|
|
33
|
+
return fail('invalid_record_metadata');
|
|
34
|
+
}
|
|
35
|
+
const maxPreviewCharacters = 200;
|
|
36
|
+
return {
|
|
37
|
+
...timestamps,
|
|
38
|
+
preview: [...input.preview.replace(/\s+/gu, ' ').trim()]
|
|
39
|
+
.slice(0, maxPreviewCharacters)
|
|
40
|
+
.join(''),
|
|
41
|
+
};
|
|
42
|
+
}
|
package/src/models/page.ts
CHANGED
|
@@ -1,37 +1,36 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type {
|
|
1
|
+
import { assetKey, assetPlaceholder } from './asset';
|
|
2
|
+
import type { SyncDefinition, SyncStep } from './definition';
|
|
3
3
|
import { fail } from './error';
|
|
4
4
|
import { canonicalJson } from './json';
|
|
5
5
|
import type { QueueLimits } from './limits';
|
|
6
|
+
import { normalizeRecordMetadata } from './metadata';
|
|
7
|
+
import type { SyncRecord } from './record';
|
|
6
8
|
import { identifier, validate } from './validation';
|
|
7
9
|
|
|
8
10
|
export function preparePage(input: {
|
|
9
|
-
page:
|
|
11
|
+
page: SyncStep;
|
|
10
12
|
definition: SyncDefinition;
|
|
11
13
|
limits: QueueLimits;
|
|
12
|
-
}):
|
|
14
|
+
}): SyncStep {
|
|
13
15
|
try {
|
|
14
16
|
const json = canonicalJson(input.page);
|
|
15
17
|
if (Buffer.byteLength(json.json) > input.limits.maxPageBytes) {
|
|
16
18
|
fail('invalid_page');
|
|
17
19
|
}
|
|
18
|
-
const page = json.value as unknown as
|
|
20
|
+
const page = json.value as unknown as SyncStep;
|
|
19
21
|
if (
|
|
20
|
-
Object.keys(page).some((key) => !['
|
|
22
|
+
Object.keys(page).some((key) => !['records', 'checkpoint', 'complete'].includes(key)) ||
|
|
21
23
|
typeof page.complete !== 'boolean'
|
|
22
24
|
) {
|
|
23
25
|
fail('invalid_page');
|
|
24
26
|
}
|
|
25
|
-
if (
|
|
26
|
-
Object.keys(page.deliverable).some((key) => key !== 'records') ||
|
|
27
|
-
!Array.isArray(page.deliverable.records) ||
|
|
28
|
-
page.deliverable.records.length > input.limits.maxPageRecords
|
|
29
|
-
) {
|
|
27
|
+
if (!Array.isArray(page.records) || page.records.length > input.limits.maxPageRecords) {
|
|
30
28
|
fail('invalid_page');
|
|
31
29
|
}
|
|
30
|
+
validatePageAssets({ page, limits: input.limits });
|
|
32
31
|
validate({ value: page.checkpoint, schema: input.definition.checkpointSchema });
|
|
33
32
|
const identities = new Set<string>();
|
|
34
|
-
for (const record of page.
|
|
33
|
+
for (const record of page.records) {
|
|
35
34
|
validateRecord({ record, definition: input.definition });
|
|
36
35
|
const key = JSON.stringify([record.kind, record.id]);
|
|
37
36
|
if (identities.has(key)) {
|
|
@@ -53,7 +52,17 @@ function validateRecord(input: { record: SyncRecord; definition: SyncDefinition
|
|
|
53
52
|
}
|
|
54
53
|
const fields =
|
|
55
54
|
record.operation === 'upsert'
|
|
56
|
-
? [
|
|
55
|
+
? [
|
|
56
|
+
'operation',
|
|
57
|
+
'kind',
|
|
58
|
+
'id',
|
|
59
|
+
'data',
|
|
60
|
+
'content',
|
|
61
|
+
'assetRefs',
|
|
62
|
+
'preview',
|
|
63
|
+
'createdAt',
|
|
64
|
+
'updatedAt',
|
|
65
|
+
]
|
|
57
66
|
: ['operation', 'kind', 'id'];
|
|
58
67
|
if (Object.keys(record).some((key) => !fields.includes(key))) {
|
|
59
68
|
fail('invalid_record');
|
|
@@ -62,8 +71,54 @@ function validateRecord(input: { record: SyncRecord; definition: SyncDefinition
|
|
|
62
71
|
if (!record.data || Array.isArray(record.data) || typeof record.data !== 'object') {
|
|
63
72
|
fail('invalid_record');
|
|
64
73
|
}
|
|
74
|
+
if (
|
|
75
|
+
record.content !== undefined &&
|
|
76
|
+
(!record.content ||
|
|
77
|
+
typeof record.content !== 'object' ||
|
|
78
|
+
Array.isArray(record.content) ||
|
|
79
|
+
record.content.format !== 'markdown' ||
|
|
80
|
+
typeof record.content.body !== 'string' ||
|
|
81
|
+
Object.keys(record.content).some((key) => !['format', 'body'].includes(key)))
|
|
82
|
+
) {
|
|
83
|
+
fail('invalid_record_content');
|
|
84
|
+
}
|
|
85
|
+
validateAssetReferences(record);
|
|
65
86
|
validate({ value: record.data, schema: definition.kinds[record.kind]! });
|
|
87
|
+
Object.assign(record, normalizeRecordMetadata(record));
|
|
66
88
|
} else if (record.operation !== 'delete') {
|
|
67
89
|
fail('invalid_record');
|
|
68
90
|
}
|
|
69
91
|
}
|
|
92
|
+
|
|
93
|
+
function validatePageAssets(input: { page: SyncStep; limits: QueueLimits }) {
|
|
94
|
+
const keys = new Set<string>();
|
|
95
|
+
for (const record of input.page.records) {
|
|
96
|
+
if (record.operation === 'upsert') {
|
|
97
|
+
for (const ref of Object.values(record.assetRefs ?? {})) {
|
|
98
|
+
keys.add(assetKey(ref));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (keys.size > input.limits.maxPageAssets) {
|
|
103
|
+
fail('invalid_assets');
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Validate declarations only. The engine does not interpret record data or content. */
|
|
108
|
+
function validateAssetReferences(record: SyncRecord): void {
|
|
109
|
+
if (record.operation !== 'upsert') {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const refs = record.assetRefs === undefined ? {} : record.assetRefs;
|
|
113
|
+
if (!refs || Array.isArray(refs) || typeof refs !== 'object') {
|
|
114
|
+
fail('invalid_asset_references');
|
|
115
|
+
}
|
|
116
|
+
for (const [key, ref] of Object.entries(refs)) {
|
|
117
|
+
assetPlaceholder(key);
|
|
118
|
+
identifier(ref.id);
|
|
119
|
+
identifier(ref.version);
|
|
120
|
+
if (Object.keys(ref).some((field) => !['id', 'version'].includes(field))) {
|
|
121
|
+
fail('invalid_asset_reference');
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import MiniSearch from 'minisearch';
|
|
2
|
+
import type { ProviderCatalogEntry } from './providers';
|
|
3
|
+
|
|
4
|
+
/** Provider metadata is fixed for the lifetime of a connector runtime. */
|
|
5
|
+
export class ProviderCatalog {
|
|
6
|
+
private index?: MiniSearch;
|
|
7
|
+
private byService?: Map<string, ProviderCatalogEntry>;
|
|
8
|
+
readonly entries: ProviderCatalogEntry[];
|
|
9
|
+
constructor(entries: ProviderCatalogEntry[]) {
|
|
10
|
+
this.entries = entries.map(
|
|
11
|
+
({ service, displayName, iconUrl, categories, scenario, authTypes }) => ({
|
|
12
|
+
service,
|
|
13
|
+
displayName,
|
|
14
|
+
iconUrl,
|
|
15
|
+
categories,
|
|
16
|
+
scenario,
|
|
17
|
+
authTypes,
|
|
18
|
+
}),
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
page(input: { q?: string; offset?: number }) {
|
|
23
|
+
const providers = this.entries;
|
|
24
|
+
const query = input.q?.trim();
|
|
25
|
+
let matches = providers;
|
|
26
|
+
if (query) {
|
|
27
|
+
if (!this.index) {
|
|
28
|
+
this.index = new MiniSearch({
|
|
29
|
+
idField: 'service',
|
|
30
|
+
fields: ['service', 'displayName', 'categoryNames', 'scenario', 'authentication'],
|
|
31
|
+
searchOptions: {
|
|
32
|
+
prefix: true,
|
|
33
|
+
combineWith: 'AND',
|
|
34
|
+
boost: { displayName: 3, service: 2 },
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
this.index.addAll(
|
|
38
|
+
providers.map((provider) => ({
|
|
39
|
+
...provider,
|
|
40
|
+
categoryNames: provider.categories.map((category) => category.displayName).join(' '),
|
|
41
|
+
authentication: provider.authTypes.map((type) => type.replaceAll('_', ' ')).join(' '),
|
|
42
|
+
})),
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
this.byService ??= new Map(providers.map((provider) => [provider.service, provider]));
|
|
46
|
+
matches = this.index.search(query).map((result) => this.byService!.get(result.id)!);
|
|
47
|
+
}
|
|
48
|
+
const pageSize = 30;
|
|
49
|
+
const offset = input.offset ?? 0;
|
|
50
|
+
return {
|
|
51
|
+
providers: matches.slice(offset, offset + pageSize),
|
|
52
|
+
total: matches.length,
|
|
53
|
+
pageSize,
|
|
54
|
+
hasMore: offset + pageSize < matches.length,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** A named segment matches one encoded identifier, never a path or traversal. */
|
|
2
|
+
export function matchesProviderPath(input: { path: string; allowed?: readonly string[] }): boolean {
|
|
3
|
+
const segments = input.path.split('/');
|
|
4
|
+
return (
|
|
5
|
+
input.allowed?.some((pattern) => {
|
|
6
|
+
const expected = pattern.split('/');
|
|
7
|
+
return (
|
|
8
|
+
expected.length === segments.length &&
|
|
9
|
+
// biome-ignore lint/complexity/useMaxParams: Array.every supplies the segment index.
|
|
10
|
+
expected.every((segment, index) => {
|
|
11
|
+
const value = segments[index]!;
|
|
12
|
+
if (!segment.startsWith(':')) {
|
|
13
|
+
return segment === value;
|
|
14
|
+
}
|
|
15
|
+
try {
|
|
16
|
+
const decoded = decodeURIComponent(value);
|
|
17
|
+
return Boolean(decoded) && !['.', '..'].includes(decoded) && !/[\\/?#%]/.test(decoded);
|
|
18
|
+
} catch {
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
})
|
|
22
|
+
);
|
|
23
|
+
}) ?? false
|
|
24
|
+
);
|
|
25
|
+
}
|
package/src/models/providers.ts
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
import type { RuntimeProviderSetup } from '@oomol-lab/open-connector';
|
|
2
|
-
export type ProviderSetup = RuntimeProviderSetup;
|
|
3
|
-
|
|
4
2
|
import type { Scope } from './identity';
|
|
3
|
+
export type ProviderSetup = Omit<RuntimeProviderSetup, 'oauthClient'> & {
|
|
4
|
+
oauthClient?: NonNullable<RuntimeProviderSetup['oauthClient']> & {
|
|
5
|
+
automaticRegistration?: boolean;
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
/** Host-registered provider code. Open Sync persists the result through encrypted Connector storage. */
|
|
10
|
+
export type OAuthClientRegistration = (input: {
|
|
11
|
+
redirectUri: string;
|
|
12
|
+
scopes: readonly string[];
|
|
13
|
+
signal: AbortSignal;
|
|
14
|
+
}) => Promise<{ clientId: string; clientSecret?: string }>;
|
|
15
|
+
|
|
5
16
|
export type ProviderScope = Scope;
|
|
6
17
|
export interface ProviderConnection {
|
|
7
18
|
id: string;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { AssetRef } from './asset';
|
|
2
|
+
import type { JsonObject } from './json';
|
|
3
|
+
import type { RecordMetadata } from './metadata';
|
|
4
|
+
|
|
5
|
+
/** Source-authored readable content; it need not duplicate or be derived from data. */
|
|
6
|
+
export interface RecordContent {
|
|
7
|
+
format: 'markdown';
|
|
8
|
+
body: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type SyncRecord =
|
|
12
|
+
| (RecordMetadata & {
|
|
13
|
+
operation: 'upsert';
|
|
14
|
+
kind: string;
|
|
15
|
+
id: string;
|
|
16
|
+
/** Source-defined JSON. The engine validates its schema but does not interpret its values. */
|
|
17
|
+
data: JsonObject;
|
|
18
|
+
content?: RecordContent;
|
|
19
|
+
/** Maps record-local placeholder keys to captured immutable assets. References need not appear in data/content. */
|
|
20
|
+
assetRefs?: Record<string, AssetRef>;
|
|
21
|
+
})
|
|
22
|
+
| { operation: 'delete'; kind: string; id: string };
|
package/src/models/registry.ts
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type DefinitionRef,
|
|
3
|
-
definitionKey,
|
|
4
|
-
type SyncDefinition,
|
|
5
|
-
type SyncRegistration,
|
|
6
|
-
} from './definition';
|
|
1
|
+
import type { SyncDefinition, SyncRegistration } from './definition';
|
|
7
2
|
import type { DestinationType } from './delivery';
|
|
8
3
|
import { fail } from './error';
|
|
9
4
|
import { canonicalJson } from './json';
|
|
@@ -19,15 +14,21 @@ export class Registry {
|
|
|
19
14
|
this.#destinations = new Map(
|
|
20
15
|
Object.entries(input.destinations).map(([name, type]) => {
|
|
21
16
|
identifier(name);
|
|
22
|
-
|
|
23
|
-
|
|
17
|
+
return [
|
|
18
|
+
name,
|
|
19
|
+
{
|
|
20
|
+
...type,
|
|
21
|
+
configSchema: structuredClone(type.configSchema),
|
|
22
|
+
setup: type.setup
|
|
23
|
+
? { ...type.setup, schema: structuredClone(type.setup.schema) }
|
|
24
|
+
: undefined,
|
|
25
|
+
},
|
|
26
|
+
];
|
|
24
27
|
}),
|
|
25
28
|
);
|
|
26
29
|
for (const registration of input.definitions) {
|
|
27
30
|
const definition = canonicalJson(registration.definition).value as unknown as SyncDefinition;
|
|
28
31
|
identifier(definition.id);
|
|
29
|
-
identifier(definition.version);
|
|
30
|
-
identifier(definition.artifactId);
|
|
31
32
|
validate({ value: definition.initialCheckpoint, schema: definition.checkpointSchema });
|
|
32
33
|
if (!Object.keys(definition.kinds).length) {
|
|
33
34
|
fail('missing_record_kinds');
|
|
@@ -35,7 +36,7 @@ export class Registry {
|
|
|
35
36
|
for (const kind of Object.keys(definition.kinds)) {
|
|
36
37
|
identifier(kind);
|
|
37
38
|
}
|
|
38
|
-
const key =
|
|
39
|
+
const key = definition.id;
|
|
39
40
|
if (this.#definitions.has(key)) {
|
|
40
41
|
fail('definition_conflict');
|
|
41
42
|
}
|
|
@@ -45,8 +46,8 @@ export class Registry {
|
|
|
45
46
|
definitions(): SyncDefinition[] {
|
|
46
47
|
return [...this.#definitions.values()].map((entry) => structuredClone(entry.definition));
|
|
47
48
|
}
|
|
48
|
-
definition(
|
|
49
|
-
const registration = this.#definitions.get(
|
|
49
|
+
definition(id: string): SyncRegistration {
|
|
50
|
+
const registration = this.#definitions.get(id);
|
|
50
51
|
if (!registration) {
|
|
51
52
|
fail('definition_unavailable');
|
|
52
53
|
}
|
|
@@ -58,10 +59,9 @@ export class Registry {
|
|
|
58
59
|
destinationTypes() {
|
|
59
60
|
return [...this.#destinations].map(([type, entry]) => ({
|
|
60
61
|
type,
|
|
61
|
-
version: entry.version,
|
|
62
62
|
name: entry.name,
|
|
63
63
|
description: entry.description,
|
|
64
|
-
|
|
64
|
+
setupSchema: structuredClone(entry.setup?.schema ?? entry.configSchema),
|
|
65
65
|
}));
|
|
66
66
|
}
|
|
67
67
|
destination(name: string): DestinationType {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { SyncError } from './error';
|
|
2
|
+
|
|
3
|
+
const firstErrorStatus = 400;
|
|
4
|
+
const lastErrorStatus = 599;
|
|
5
|
+
|
|
6
|
+
/** Sources normalize provider-specific failures to HTTP semantics. No bodies or secrets are retained. */
|
|
7
|
+
export class SourceHttpError extends SyncError {
|
|
8
|
+
declare readonly status: number;
|
|
9
|
+
|
|
10
|
+
constructor(input: { status: number }) {
|
|
11
|
+
if (!isErrorStatus(input.status)) {
|
|
12
|
+
throw new TypeError('SourceHttpError requires an HTTP error status (400–599).');
|
|
13
|
+
}
|
|
14
|
+
super({
|
|
15
|
+
code: `source_http_${input.status}`,
|
|
16
|
+
message: `Source returned HTTP ${input.status}.`,
|
|
17
|
+
status: input.status,
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function isErrorStatus(value: unknown): value is number {
|
|
23
|
+
return (
|
|
24
|
+
Number.isInteger(value) && Number(value) >= firstErrorStatus && Number(value) <= lastErrorStatus
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function retryableStatus(status: number): boolean {
|
|
29
|
+
const serverError = 500;
|
|
30
|
+
const requestTimeout = 408;
|
|
31
|
+
const conflict = 409;
|
|
32
|
+
const tooEarly = 425;
|
|
33
|
+
const rateLimited = 429;
|
|
34
|
+
return (
|
|
35
|
+
status >= serverError || [requestTimeout, conflict, tooEarly, rateLimited].includes(status)
|
|
36
|
+
);
|
|
37
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { ConnectionRef } from './definition';
|
|
2
|
+
import type { Scope } from './identity';
|
|
3
|
+
import type { JsonObject, JsonValue } from './json';
|
|
4
|
+
|
|
5
|
+
export interface Sync {
|
|
6
|
+
id: string;
|
|
7
|
+
ownerId: string;
|
|
8
|
+
definition: string;
|
|
9
|
+
connection?: ConnectionRef;
|
|
10
|
+
config: JsonObject;
|
|
11
|
+
destination: { type: string; config: JsonObject };
|
|
12
|
+
enabled: boolean;
|
|
13
|
+
checkpoint: JsonValue;
|
|
14
|
+
intervalMs: number;
|
|
15
|
+
nextDueAt: number;
|
|
16
|
+
status: SyncStatus;
|
|
17
|
+
errorCode: string | null;
|
|
18
|
+
}
|
|
19
|
+
export interface CreateSync extends Scope {
|
|
20
|
+
definition: string;
|
|
21
|
+
connection?: ConnectionRef;
|
|
22
|
+
config: JsonObject;
|
|
23
|
+
destination: { type: string; input: JsonObject };
|
|
24
|
+
intervalMs?: number;
|
|
25
|
+
enabled?: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type SyncStatus =
|
|
29
|
+
| 'ready'
|
|
30
|
+
| 'running'
|
|
31
|
+
| 'retrying'
|
|
32
|
+
| 'waiting_for_capacity'
|
|
33
|
+
| 'disabled'
|
|
34
|
+
| 'succeeded'
|
|
35
|
+
| 'interrupted';
|
|
36
|
+
|
|
37
|
+
/** One scan through complete=true, including retries. Counts cover committed pages, not delivery acceptance. */
|
|
38
|
+
export interface SyncPoll {
|
|
39
|
+
id: number;
|
|
40
|
+
startedAt: number;
|
|
41
|
+
completedAt: number | null;
|
|
42
|
+
state: SyncStatus;
|
|
43
|
+
errorCode: string | null;
|
|
44
|
+
recordsProcessed: number;
|
|
45
|
+
recordsQueued: number;
|
|
46
|
+
}
|
|
47
|
+
/** Host-visible status; configuration, checkpoints and execution fencing remain private. */
|
|
48
|
+
export interface SyncSummary {
|
|
49
|
+
id: string;
|
|
50
|
+
definition: string;
|
|
51
|
+
destinationType: string;
|
|
52
|
+
connection?: ConnectionRef;
|
|
53
|
+
enabled: boolean;
|
|
54
|
+
intervalMs: number;
|
|
55
|
+
nextDueAt: number;
|
|
56
|
+
status: SyncStatus;
|
|
57
|
+
errorCode: string | null;
|
|
58
|
+
}
|
|
59
|
+
export function summarizeSync(sync: Sync): SyncSummary {
|
|
60
|
+
return {
|
|
61
|
+
id: sync.id,
|
|
62
|
+
definition: sync.definition,
|
|
63
|
+
destinationType: sync.destination.type,
|
|
64
|
+
connection: sync.connection,
|
|
65
|
+
enabled: sync.enabled,
|
|
66
|
+
intervalMs: sync.intervalMs,
|
|
67
|
+
nextDueAt: sync.nextDueAt,
|
|
68
|
+
status: sync.status,
|
|
69
|
+
errorCode: sync.errorCode,
|
|
70
|
+
};
|
|
71
|
+
}
|
package/src/models/validation.ts
CHANGED
|
@@ -11,7 +11,7 @@ export function identifier(value: unknown): asserts value is string {
|
|
|
11
11
|
export function validate(input: { value: unknown; schema: Schema }): JsonValue {
|
|
12
12
|
try {
|
|
13
13
|
const json = canonicalJson(input.value);
|
|
14
|
-
// The validator mutates schemas while compiling;
|
|
14
|
+
// The validator mutates schemas while compiling; host-provided schemas remain unchanged.
|
|
15
15
|
if (
|
|
16
16
|
!new Validator(structuredClone(input.schema), '2020-12', false).validate(json.value).valid
|
|
17
17
|
) {
|
package/src/open-sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
|
-
import { type ProviderApi, providerApi, type SyncApi
|
|
2
|
+
import { type ProviderApi, providerApi, type SyncApi } from './api';
|
|
3
3
|
import { createConnectorClient } from './connector/client';
|
|
4
4
|
import { loadProviderKey } from './connector/encryption-key';
|
|
5
5
|
import { connectorManagement } from './connector/management';
|
|
@@ -11,26 +11,40 @@ import type { DestinationType } from './models/delivery';
|
|
|
11
11
|
import { fail } from './models/error';
|
|
12
12
|
import type { Scope } from './models/identity';
|
|
13
13
|
import { defaultTiming, type QueueLimits } from './models/limits';
|
|
14
|
+
import type { OAuthClientRegistration } from './models/providers';
|
|
14
15
|
import { SqliteProviders } from './repositories/providers/sqlite';
|
|
15
16
|
import { createSyncRuntime } from './runtime';
|
|
16
17
|
import { ProviderService } from './services/providers/service';
|
|
17
18
|
|
|
18
19
|
export type { Scope } from './models/identity';
|
|
19
|
-
export type {
|
|
20
|
+
export type {
|
|
21
|
+
OAuthClientRegistration,
|
|
22
|
+
ProviderCatalogEntry,
|
|
23
|
+
ProviderSetup,
|
|
24
|
+
} from './models/providers';
|
|
20
25
|
|
|
21
26
|
export interface OpenSyncOptions {
|
|
22
27
|
definitions: readonly SyncRegistration[];
|
|
23
28
|
destinationTypes: Readonly<Record<string, DestinationType>>;
|
|
24
29
|
limits?: Partial<QueueLimits>;
|
|
25
30
|
executionTimeoutMs?: number;
|
|
31
|
+
sourceConcurrency?: number;
|
|
32
|
+
deliveryConcurrency?: number;
|
|
26
33
|
onEvent?: Logger;
|
|
27
34
|
dataDirectory: string;
|
|
35
|
+
assetDirectory?: string;
|
|
28
36
|
/** Absolute URL where the host mounts fetch(), including its path prefix. */
|
|
29
37
|
publicUrl: string;
|
|
30
38
|
/** Host authentication, ownership and CSRF policy for management HTTP requests. */
|
|
31
39
|
authorize(request: Request): Scope | null | Promise<Scope | null>;
|
|
32
40
|
/** OAuth application settings are instance-wide, so require the host's administrator policy. */
|
|
33
41
|
canConfigureProviders(scope: Scope): Promise<boolean>;
|
|
42
|
+
/** Optional automatic OAuth client setup, selected by the host; never runs on catalog reads. */
|
|
43
|
+
oauthClientRegistrations?: Readonly<Record<string, OAuthClientRegistration>>;
|
|
44
|
+
/** Called after an owned connection is saved, including a retried OAuth completion. */
|
|
45
|
+
onProviderConnected?(
|
|
46
|
+
input: Scope & { connection: { id: string; service: string } },
|
|
47
|
+
): Promise<void>;
|
|
34
48
|
/** Final host UI location after Open Sync completes authorization. */
|
|
35
49
|
authorizationRedirect?(input: { service: string; outcome: 'connected' | 'failed' }): string;
|
|
36
50
|
}
|
|
@@ -103,11 +117,9 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
|
|
|
103
117
|
destinationTypes: options.destinationTypes,
|
|
104
118
|
limits: options.limits,
|
|
105
119
|
onEvent: options.onEvent,
|
|
106
|
-
timing:
|
|
107
|
-
timeoutMs: options.executionTimeoutMs ?? defaultTiming.timeoutMs,
|
|
108
|
-
leaseMs: (options.executionTimeoutMs ?? defaultTiming.timeoutMs) + defaultTiming.timeoutMs,
|
|
109
|
-
},
|
|
120
|
+
timing: executionTiming(options),
|
|
110
121
|
databasePath: join(options.dataDirectory, 'sync.db'),
|
|
122
|
+
assetDirectory: options.assetDirectory ?? join(options.dataDirectory, 'queued-assets'),
|
|
111
123
|
connector: {
|
|
112
124
|
async bind(input) {
|
|
113
125
|
const owned = await repository.connection({ ...input, id: input.connection.id });
|
|
@@ -126,10 +138,12 @@ export async function createOpenSync(options: OpenSyncOptions): Promise<OpenSync
|
|
|
126
138
|
connector: management,
|
|
127
139
|
signal: lifetime.signal,
|
|
128
140
|
canConfigure: options.canConfigureProviders,
|
|
141
|
+
registrations: options.oauthClientRegistrations,
|
|
142
|
+
onConnected: options.onProviderConnected,
|
|
129
143
|
returnUrl: (input) =>
|
|
130
144
|
`${publicUrl}/providers/${encodeURIComponent(input.service)}/return/${input.id}`,
|
|
131
145
|
});
|
|
132
|
-
const api =
|
|
146
|
+
const api = engine.api;
|
|
133
147
|
const http = createHttpApp({
|
|
134
148
|
prefix,
|
|
135
149
|
api,
|
|
@@ -195,3 +209,13 @@ export interface OpenSyncRuntime {
|
|
|
195
209
|
close(): Promise<void>;
|
|
196
210
|
}
|
|
197
211
|
export type { ProviderApi, SyncApi } from './api';
|
|
212
|
+
|
|
213
|
+
function executionTiming(options: OpenSyncOptions) {
|
|
214
|
+
const timeoutMs = options.executionTimeoutMs ?? defaultTiming.timeoutMs;
|
|
215
|
+
return {
|
|
216
|
+
timeoutMs,
|
|
217
|
+
leaseMs: timeoutMs + defaultTiming.timeoutMs,
|
|
218
|
+
sourceConcurrency: options.sourceConcurrency ?? defaultTiming.sourceConcurrency,
|
|
219
|
+
deliveryConcurrency: options.deliveryConcurrency ?? defaultTiming.deliveryConcurrency,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
@@ -1,17 +1,25 @@
|
|
|
1
|
-
import type { SyncDefinition,
|
|
1
|
+
import type { SyncDefinition, SyncStep } from '../../models/definition';
|
|
2
2
|
import type { Scope } from '../../models/identity';
|
|
3
|
-
import type {
|
|
3
|
+
import type { Sync, SyncStatus } from '../../models/sync';
|
|
4
4
|
|
|
5
|
-
export interface
|
|
6
|
-
|
|
7
|
-
installation: Installation;
|
|
8
|
-
workerId: string;
|
|
5
|
+
export interface AcquisitionLease extends Scope {
|
|
6
|
+
sync: Sync;
|
|
9
7
|
generation: number;
|
|
10
|
-
|
|
8
|
+
force: boolean;
|
|
9
|
+
failureCount: number;
|
|
11
10
|
}
|
|
12
11
|
export interface AcquisitionRepository {
|
|
13
|
-
|
|
12
|
+
capacityReleased(): void;
|
|
13
|
+
nextDue(): number | undefined;
|
|
14
|
+
claim(leaseMs: number): AcquisitionLease | undefined;
|
|
14
15
|
hasCapacity(): boolean;
|
|
15
|
-
commit(input: { lease:
|
|
16
|
-
finish(input: {
|
|
16
|
+
commit(input: { lease: AcquisitionLease; page: SyncStep; definition: SyncDefinition }): void;
|
|
17
|
+
finish(input: {
|
|
18
|
+
lease: AcquisitionLease;
|
|
19
|
+
state: Exclude<SyncStatus, 'running' | 'disabled'>;
|
|
20
|
+
errorCode?: string;
|
|
21
|
+
delay: number;
|
|
22
|
+
failureCount?: number;
|
|
23
|
+
pause?: boolean;
|
|
24
|
+
}): void;
|
|
17
25
|
}
|