nucleus-core-ts 0.9.781 → 0.9.783
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +1 -1
- package/dist/fe/components/IntegrationsPage/components/ConnectionAuthFields.d.ts +2 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionAuthFields.js +85 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionBasicFields.d.ts +2 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionBasicFields.js +165 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionForm.d.ts +10 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionForm.js +112 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionList.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionList.js +184 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionOauthFields.d.ts +2 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionOauthFields.js +131 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionSettingsTab.d.ts +21 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionSettingsTab.js +49 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionTabs.d.ts +12 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionTabs.js +95 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionTest.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/ConnectionTest.js +116 -0
- package/dist/fe/components/IntegrationsPage/components/CredentialFields.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/CredentialFields.js +171 -0
- package/dist/fe/components/IntegrationsPage/components/CredentialSlotRow.d.ts +13 -0
- package/dist/fe/components/IntegrationsPage/components/CredentialSlotRow.js +63 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointForm.d.ts +11 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointForm.js +166 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointFormFields.d.ts +19 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointFormFields.js +162 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointList.d.ts +12 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointList.js +205 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointSample.d.ts +6 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointSample.js +204 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointsTab.d.ts +5 -0
- package/dist/fe/components/IntegrationsPage/components/EndpointsTab.js +101 -0
- package/dist/fe/components/IntegrationsPage/components/FieldMappingRow.d.ts +17 -0
- package/dist/fe/components/IntegrationsPage/components/FieldMappingRow.js +179 -0
- package/dist/fe/components/IntegrationsPage/components/FieldMappingRows.d.ts +11 -0
- package/dist/fe/components/IntegrationsPage/components/FieldMappingRows.js +115 -0
- package/dist/fe/components/IntegrationsPage/components/IntegrationsPage.d.ts +2 -0
- package/dist/fe/components/IntegrationsPage/components/IntegrationsPage.js +175 -0
- package/dist/fe/components/IntegrationsPage/components/MappingEditor.d.ts +15 -0
- package/dist/fe/components/IntegrationsPage/components/MappingEditor.js +193 -0
- package/dist/fe/components/IntegrationsPage/components/MappingList.d.ts +11 -0
- package/dist/fe/components/IntegrationsPage/components/MappingList.js +191 -0
- package/dist/fe/components/IntegrationsPage/components/MappingWriteModeFields.d.ts +19 -0
- package/dist/fe/components/IntegrationsPage/components/MappingWriteModeFields.js +150 -0
- package/dist/fe/components/IntegrationsPage/components/MappingsTab.d.ts +5 -0
- package/dist/fe/components/IntegrationsPage/components/MappingsTab.js +154 -0
- package/dist/fe/components/IntegrationsPage/components/PaginationFields.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/PaginationFields.js +218 -0
- package/dist/fe/components/IntegrationsPage/components/ParamBindingRow.d.ts +24 -0
- package/dist/fe/components/IntegrationsPage/components/ParamBindingRow.js +180 -0
- package/dist/fe/components/IntegrationsPage/components/ParamBindingRows.d.ts +33 -0
- package/dist/fe/components/IntegrationsPage/components/ParamBindingRows.js +100 -0
- package/dist/fe/components/IntegrationsPage/components/Primitives.d.ts +65 -0
- package/dist/fe/components/IntegrationsPage/components/Primitives.js +94 -0
- package/dist/fe/components/IntegrationsPage/components/RunConfirmStep.d.ts +10 -0
- package/dist/fe/components/IntegrationsPage/components/RunConfirmStep.js +48 -0
- package/dist/fe/components/IntegrationsPage/components/RunHistory.d.ts +10 -0
- package/dist/fe/components/IntegrationsPage/components/RunHistory.js +124 -0
- package/dist/fe/components/IntegrationsPage/components/RunHistoryRow.d.ts +7 -0
- package/dist/fe/components/IntegrationsPage/components/RunHistoryRow.js +129 -0
- package/dist/fe/components/IntegrationsPage/components/RunPanel.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/RunPanel.js +160 -0
- package/dist/fe/components/IntegrationsPage/components/RunPlanIssues.d.ts +5 -0
- package/dist/fe/components/IntegrationsPage/components/RunPlanIssues.js +43 -0
- package/dist/fe/components/IntegrationsPage/components/RunPlanSummary.d.ts +5 -0
- package/dist/fe/components/IntegrationsPage/components/RunPlanSummary.js +92 -0
- package/dist/fe/components/IntegrationsPage/components/RunProgressBar.d.ts +8 -0
- package/dist/fe/components/IntegrationsPage/components/RunProgressBar.js +77 -0
- package/dist/fe/components/IntegrationsPage/components/RunRevertConflicts.d.ts +13 -0
- package/dist/fe/components/IntegrationsPage/components/RunRevertConflicts.js +65 -0
- package/dist/fe/components/IntegrationsPage/components/RunsTab.d.ts +13 -0
- package/dist/fe/components/IntegrationsPage/components/RunsTab.js +39 -0
- package/dist/fe/components/IntegrationsPage/components/Skeletons.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/Skeletons.js +139 -0
- package/dist/fe/components/IntegrationsPage/components/authOptions.d.ts +29 -0
- package/dist/fe/components/IntegrationsPage/components/authOptions.js +71 -0
- package/dist/fe/components/IntegrationsPage/components/connectionDraft.d.ts +64 -0
- package/dist/fe/components/IntegrationsPage/components/connectionDraft.js +112 -0
- package/dist/fe/components/IntegrationsPage/components/credentialSlots.d.ts +25 -0
- package/dist/fe/components/IntegrationsPage/components/credentialSlots.js +67 -0
- package/dist/fe/components/IntegrationsPage/components/fieldMappingOptions.d.ts +16 -0
- package/dist/fe/components/IntegrationsPage/components/fieldMappingOptions.js +41 -0
- package/dist/fe/components/IntegrationsPage/components/formSupport.d.ts +19 -0
- package/dist/fe/components/IntegrationsPage/components/formSupport.js +25 -0
- package/dist/fe/components/IntegrationsPage/components/mappingDraft.d.ts +40 -0
- package/dist/fe/components/IntegrationsPage/components/mappingDraft.js +44 -0
- package/dist/fe/components/IntegrationsPage/components/pathParams.d.ts +9 -0
- package/dist/fe/components/IntegrationsPage/components/pathParams.js +15 -0
- package/dist/fe/components/IntegrationsPage/components/runFormat.d.ts +33 -0
- package/dist/fe/components/IntegrationsPage/components/runFormat.js +85 -0
- package/dist/fe/components/IntegrationsPage/components/useInfiniteList.d.ts +40 -0
- package/dist/fe/components/IntegrationsPage/components/useInfiniteList.js +110 -0
- package/dist/fe/components/IntegrationsPage/index.d.ts +18 -0
- package/dist/fe/components/IntegrationsPage/index.js +11 -0
- package/dist/fe/components/IntegrationsPage/store/index.d.ts +2 -0
- package/dist/fe/components/IntegrationsPage/store/index.js +124 -0
- package/dist/fe/components/IntegrationsPage/store/state.d.ts +80 -0
- package/dist/fe/components/IntegrationsPage/store/state.js +32 -0
- package/dist/fe/components/IntegrationsPage/theme/index.d.ts +103 -0
- package/dist/fe/components/IntegrationsPage/theme/index.js +113 -0
- package/dist/fe/components/IntegrationsPage/types/actions.d.ts +91 -0
- package/dist/fe/components/IntegrationsPage/types/actions.js +1 -0
- package/dist/fe/components/IntegrationsPage/types/index.d.ts +3 -0
- package/dist/fe/components/IntegrationsPage/types/index.js +3 -0
- package/dist/fe/components/IntegrationsPage/types/records.d.ts +139 -0
- package/dist/fe/components/IntegrationsPage/types/records.js +11 -0
- package/dist/fe/components/IntegrationsPage/types/results.d.ts +52 -0
- package/dist/fe/components/IntegrationsPage/types/results.js +4 -0
- package/dist/fe/index.d.ts +2 -0
- package/dist/fe/index.js +1 -0
- package/dist/index.js +14 -14
- package/dist/src/Client/ApiCaller/system-tables.d.ts +505 -0
- package/dist/src/Client/Proxy/wsProxy.js +25 -1
- package/dist/src/ElysiaPlugin/routes/backup/types.d.ts +3 -1
- package/dist/src/ElysiaPlugin/routes/integrations/index.d.ts +49 -0
- package/dist/src/ElysiaPlugin/routes/integrations/inspect.d.ts +87 -0
- package/dist/src/ElysiaPlugin/routes/integrations/inspectRoutes.d.ts +41 -0
- package/dist/src/ElysiaPlugin/routes/integrations/repository.d.ts +88 -0
- package/dist/src/ElysiaPlugin/routes/integrations/types.d.ts +22 -0
- package/dist/src/ElysiaPlugin/routes/pubsub/frame.d.ts +37 -0
- package/dist/src/ElysiaPlugin/routes/pubsub/frame.test.d.ts +1 -0
- package/dist/src/Services/Backup/BackupService.d.ts +16 -8
- package/dist/src/Services/Backup/types.d.ts +25 -0
- package/dist/src/Services/Integrations/bindings.d.ts +116 -0
- package/dist/src/Services/Integrations/bindings.test.d.ts +1 -0
- package/dist/src/Services/Integrations/collect.d.ts +68 -0
- package/dist/src/Services/Integrations/collect.test.d.ts +1 -0
- package/dist/src/Services/Integrations/fetcher.d.ts +100 -0
- package/dist/src/Services/Integrations/fetcher.test.d.ts +1 -0
- package/dist/src/Services/Integrations/lock.d.ts +77 -0
- package/dist/src/Services/Integrations/lock.test.d.ts +1 -0
- package/dist/src/Services/Integrations/mapping.d.ts +63 -0
- package/dist/src/Services/Integrations/mapping.test.d.ts +1 -0
- package/dist/src/Services/Integrations/oauth.d.ts +89 -0
- package/dist/src/Services/Integrations/oauth.test.d.ts +1 -0
- package/dist/src/Services/Integrations/pagination.d.ts +97 -0
- package/dist/src/Services/Integrations/pagination.test.d.ts +1 -0
- package/dist/src/Services/Integrations/plan.d.ts +74 -0
- package/dist/src/Services/Integrations/plan.test.d.ts +1 -0
- package/dist/src/Services/Integrations/revert.d.ts +78 -0
- package/dist/src/Services/Integrations/revert.test.d.ts +1 -0
- package/dist/src/Services/Integrations/runner.d.ts +126 -0
- package/dist/src/Services/Integrations/runner.test.d.ts +1 -0
- package/dist/src/Services/Integrations/types.d.ts +208 -0
- package/dist/src/Services/Integrations/writer.d.ts +105 -0
- package/dist/src/Services/Integrations/writer.test.d.ts +1 -0
- package/dist/src/Utils/cron.d.ts +65 -0
- package/dist/src/Utils/cron.test.d.ts +1 -0
- package/dist/src/types.d.ts +67 -1
- package/package.json +1 -1
- package/schemas/config.nucleus.json +43 -0
- package/schemas/table.schema.json +2 -1
- package/scripts/generate-schema.test.ts +78 -0
- package/scripts/generate-schema.ts +43 -5
- package/src/system.tables.json +362 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Making sure one mapping imports at a time, across every replica.
|
|
3
|
+
*
|
|
4
|
+
* Two runs of the same mapping overlapping is not a performance problem, it is a
|
|
5
|
+
* correctness one: both read the same "what is already here", both decide the
|
|
6
|
+
* same rows are missing, and both act on that decision. Under a `delete` policy
|
|
7
|
+
* that removes rows the second run has already re-created.
|
|
8
|
+
*
|
|
9
|
+
* ## Why not `RedisManager.acquireLock` directly
|
|
10
|
+
*
|
|
11
|
+
* That helper stores a literal `'1'`, so `releaseLock` cannot tell whose lock it
|
|
12
|
+
* is deleting. If run A stalls long enough for its lock to expire and run B takes
|
|
13
|
+
* it, A finishing normally then deletes B's lock and a third run walks straight
|
|
14
|
+
* in. The holder here writes a token and checks it before releasing, so a lock it
|
|
15
|
+
* no longer owns is left alone.
|
|
16
|
+
*
|
|
17
|
+
* ## Why it renews
|
|
18
|
+
*
|
|
19
|
+
* A fixed TTL has to be either longer than the slowest import — in which case a
|
|
20
|
+
* crashed pod blocks the mapping for that long — or shorter, in which case a
|
|
21
|
+
* healthy long import loses its lock halfway. Renewing on a timer gives both: the
|
|
22
|
+
* TTL stays short, and a holder that dies simply stops renewing.
|
|
23
|
+
*/
|
|
24
|
+
import type { Logger } from '../Logger';
|
|
25
|
+
type RedisResult<T> = {
|
|
26
|
+
success: true;
|
|
27
|
+
data: T;
|
|
28
|
+
} | {
|
|
29
|
+
success: false;
|
|
30
|
+
error: string;
|
|
31
|
+
};
|
|
32
|
+
/** The slice of the Redis manager this needs. */
|
|
33
|
+
export type LockStore = {
|
|
34
|
+
create<T>(key: string, value: T, ttlSeconds?: number): Promise<RedisResult<'OK'>>;
|
|
35
|
+
read<T>(key: string): Promise<RedisResult<T | null>>;
|
|
36
|
+
update<T>(key: string, value: T, preserveTtl?: boolean): Promise<RedisResult<'OK'>>;
|
|
37
|
+
remove(key: string): Promise<RedisResult<number>>;
|
|
38
|
+
exists(key: string): Promise<RedisResult<boolean>>;
|
|
39
|
+
};
|
|
40
|
+
export type LockHandle = {
|
|
41
|
+
/** Stops renewing and releases, but only if this holder still owns the lock. */
|
|
42
|
+
release: () => Promise<void>;
|
|
43
|
+
/** False once a renewal has found the lock owned by somebody else. */
|
|
44
|
+
isHeld: () => boolean;
|
|
45
|
+
token: string;
|
|
46
|
+
};
|
|
47
|
+
export type AcquireResult = {
|
|
48
|
+
ok: true;
|
|
49
|
+
handle: LockHandle;
|
|
50
|
+
} | {
|
|
51
|
+
ok: false;
|
|
52
|
+
reason: 'held';
|
|
53
|
+
heldSince: number | null;
|
|
54
|
+
} | {
|
|
55
|
+
ok: false;
|
|
56
|
+
reason: 'unavailable';
|
|
57
|
+
error: string;
|
|
58
|
+
};
|
|
59
|
+
/** Short by design — a dead holder should block the mapping for seconds, not hours. */
|
|
60
|
+
export declare const LOCK_TTL_SECONDS = 60;
|
|
61
|
+
/** Renew well inside the TTL so one slow renewal does not lose the lock. */
|
|
62
|
+
export declare const RENEW_INTERVAL_MS = 20000;
|
|
63
|
+
export declare const lockKeyFor: (mappingId: string) => string;
|
|
64
|
+
/**
|
|
65
|
+
* Takes the lock for a mapping, or reports who holds it.
|
|
66
|
+
*
|
|
67
|
+
* Redis being unreachable is reported as `unavailable`, NOT as "free". Treating a
|
|
68
|
+
* broken lock service as an open door is how two runs end up deleting each
|
|
69
|
+
* other's records — the caller decides whether to proceed, and for a destructive
|
|
70
|
+
* write mode the answer should be no.
|
|
71
|
+
*/
|
|
72
|
+
export declare function acquireRunLock(store: LockStore, mappingId: string, deps?: {
|
|
73
|
+
logger?: Logger;
|
|
74
|
+
now?: () => number;
|
|
75
|
+
ttlSeconds?: number;
|
|
76
|
+
}): Promise<AcquireResult>;
|
|
77
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning a third-party record into one of our rows. Pure — no I/O, no database,
|
|
3
|
+
* no config. Everything here is a function of its arguments, so the rules that
|
|
4
|
+
* decide what lands in a column can be tested without a server or a source.
|
|
5
|
+
*/
|
|
6
|
+
import type { FieldMapping, MappedRecord, MappingTransform, RunIssue } from './types';
|
|
7
|
+
/** Reads a possibly-nested value by dot-path (`a.b.c`). */
|
|
8
|
+
export declare function getByPath(obj: unknown, path: string): unknown;
|
|
9
|
+
/**
|
|
10
|
+
* Pulls the record array out of a response body.
|
|
11
|
+
*
|
|
12
|
+
* `rootPath` always wins. Without one we look for the usual envelope keys, then
|
|
13
|
+
* fall back to a bounded search for the first array of objects anywhere in the
|
|
14
|
+
* body — plenty of APIs wrap their list under a name nobody would guess.
|
|
15
|
+
*
|
|
16
|
+
* A rootPath that points at a single object yields that object as one record,
|
|
17
|
+
* because "fetch this one thing" is a legitimate endpoint.
|
|
18
|
+
*
|
|
19
|
+
* Never throws. An unrecognisable body yields no records, and the caller decides
|
|
20
|
+
* whether that is an error — for a root endpoint it is; for a child call in a
|
|
21
|
+
* fan-out it may just mean this particular parent has none.
|
|
22
|
+
*/
|
|
23
|
+
export declare function extractRecords(body: unknown, rootPath?: string | null): unknown[];
|
|
24
|
+
/**
|
|
25
|
+
* The outcome of one transform.
|
|
26
|
+
*
|
|
27
|
+
* A failed conversion is reported rather than silently becoming null. "The HR
|
|
28
|
+
* system sent `N/A` where a number was expected" and "the field was empty" are
|
|
29
|
+
* different facts, and only the first one is worth an operator's attention — the
|
|
30
|
+
* old engine collapsed both to null and neither reached anyone.
|
|
31
|
+
*/
|
|
32
|
+
export type TransformResult = {
|
|
33
|
+
ok: true;
|
|
34
|
+
value: unknown;
|
|
35
|
+
} | {
|
|
36
|
+
ok: false;
|
|
37
|
+
reason: string;
|
|
38
|
+
raw: unknown;
|
|
39
|
+
};
|
|
40
|
+
/** Applies one transform. `none`/unknown returns the value untouched. */
|
|
41
|
+
export declare function applyTransform(value: unknown, transform?: MappingTransform): TransformResult;
|
|
42
|
+
/**
|
|
43
|
+
* Maps ONE external record onto a row.
|
|
44
|
+
*
|
|
45
|
+
* `parent`, when present, is exposed to mappings under the `__parent` prefix so a
|
|
46
|
+
* child record fetched per-parent can carry its parent's fields — e.g. the company
|
|
47
|
+
* name that produced this employee. Without it a fan-out loses the very thing that
|
|
48
|
+
* distinguishes one batch from the next.
|
|
49
|
+
*/
|
|
50
|
+
export declare function mapRecord(external: unknown, mappings: readonly FieldMapping[], parent?: unknown): {
|
|
51
|
+
row: Record<string, unknown>;
|
|
52
|
+
problems: string[];
|
|
53
|
+
};
|
|
54
|
+
/** Maps a batch, carrying dedup keys and per-record problems out with it. */
|
|
55
|
+
export declare function buildMappedRecords(externalRecords: readonly unknown[], mappings: readonly FieldMapping[], dedupTargetField?: string | null, parents?: readonly unknown[]): {
|
|
56
|
+
records: MappedRecord[];
|
|
57
|
+
issues: RunIssue[];
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* Dot-paths of a record's leaf fields, for the mapping screen's field picker.
|
|
61
|
+
* Capped in both breadth and depth — this feeds a dropdown, not an analysis.
|
|
62
|
+
*/
|
|
63
|
+
export declare function flattenKeys(obj: unknown, prefix?: string, out?: string[], depth?: number): string[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minting and reusing OAuth2 bearer tokens for a source.
|
|
3
|
+
*
|
|
4
|
+
* Supports the two grants a server-to-server integration actually uses:
|
|
5
|
+
* `client_credentials`, and the resource-owner `password` grant that Keycloak
|
|
6
|
+
* deployments still hand out for service accounts. An access token is short-lived
|
|
7
|
+
* by design, so it is cached and renewed rather than re-minted per request.
|
|
8
|
+
*
|
|
9
|
+
* ## The cache key includes the credentials
|
|
10
|
+
*
|
|
11
|
+
* Keying only by source id — which is what the engine this replaces did — means
|
|
12
|
+
* that after an operator corrects a wrong password, the cached token minted with
|
|
13
|
+
* the OLD one keeps being used until it expires. The integration goes on failing
|
|
14
|
+
* for up to an hour after it was fixed, and nothing on screen explains why.
|
|
15
|
+
* Hashing the credentials into the key makes a corrected credential a cache miss
|
|
16
|
+
* immediately.
|
|
17
|
+
*
|
|
18
|
+
* The hash is one-way: the cache key never holds the secret itself.
|
|
19
|
+
*/
|
|
20
|
+
import { type FetchImpl } from './fetcher';
|
|
21
|
+
export type OAuthGrant = 'password' | 'client_credentials';
|
|
22
|
+
export type OAuthConfig = {
|
|
23
|
+
sourceId: string;
|
|
24
|
+
tokenUrl: string;
|
|
25
|
+
grantType: OAuthGrant;
|
|
26
|
+
clientId?: string | null;
|
|
27
|
+
/** Decrypted. */
|
|
28
|
+
clientSecret?: string | null;
|
|
29
|
+
username?: string | null;
|
|
30
|
+
/** Decrypted. */
|
|
31
|
+
password?: string | null;
|
|
32
|
+
scope?: string | null;
|
|
33
|
+
/** How client credentials are presented: an HTTP Basic header, or form fields. */
|
|
34
|
+
clientAuth?: 'basic' | 'body';
|
|
35
|
+
tlsVerify?: boolean;
|
|
36
|
+
timeoutMs?: number;
|
|
37
|
+
};
|
|
38
|
+
export type CachedToken = {
|
|
39
|
+
accessToken: string;
|
|
40
|
+
expiresAt: number;
|
|
41
|
+
refreshToken?: string;
|
|
42
|
+
refreshExpiresAt?: number;
|
|
43
|
+
/** Fingerprint of the credentials this token was minted with. */
|
|
44
|
+
fingerprint: string;
|
|
45
|
+
};
|
|
46
|
+
export type TokenResult = {
|
|
47
|
+
ok: true;
|
|
48
|
+
accessToken: string;
|
|
49
|
+
fromCache: boolean;
|
|
50
|
+
} | {
|
|
51
|
+
ok: false;
|
|
52
|
+
error: string;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* One-way fingerprint of everything that changes which token a config produces.
|
|
56
|
+
*
|
|
57
|
+
* The token URL is included: pointing a source at a different realm must not
|
|
58
|
+
* reuse the previous realm's token.
|
|
59
|
+
*/
|
|
60
|
+
export declare function credentialFingerprint(config: OAuthConfig): string;
|
|
61
|
+
/** In-memory token cache. Per process — a token is not worth sharing across pods. */
|
|
62
|
+
export declare class TokenCache {
|
|
63
|
+
private readonly entries;
|
|
64
|
+
/**
|
|
65
|
+
* The stored token for a source, or null when there is none or the credentials
|
|
66
|
+
* have changed since it was minted. Expiry is NOT checked here — the caller
|
|
67
|
+
* distinguishes "expired but refreshable" from "unusable", and this cannot.
|
|
68
|
+
*/
|
|
69
|
+
get(sourceId: string, fingerprint: string): CachedToken | null;
|
|
70
|
+
set(sourceId: string, token: CachedToken): void;
|
|
71
|
+
/** Drops a source's token — used by "test connection", which must mint fresh. */
|
|
72
|
+
invalidate(sourceId: string): void;
|
|
73
|
+
clear(): void;
|
|
74
|
+
}
|
|
75
|
+
/** Builds the form body and headers for one token request. */
|
|
76
|
+
export declare function buildTokenRequest(config: OAuthConfig, form: URLSearchParams): {
|
|
77
|
+
headers: Record<string, string>;
|
|
78
|
+
body: string;
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* Returns a usable bearer for a source.
|
|
82
|
+
*
|
|
83
|
+
* Order: a cached token that is still valid → a refresh exchange (which does not
|
|
84
|
+
* re-send the service account's password) → a full grant.
|
|
85
|
+
*/
|
|
86
|
+
export declare function getAccessToken(config: OAuthConfig, cache: TokenCache, deps?: {
|
|
87
|
+
fetchImpl?: FetchImpl;
|
|
88
|
+
now?: () => number;
|
|
89
|
+
}): Promise<TokenResult>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Walking a paged endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Pure: given the profile an operator chose, the walk so far, and what the last
|
|
5
|
+
* response contained, this decides whether to ask for another page and how. No
|
|
6
|
+
* fetching happens here.
|
|
7
|
+
*
|
|
8
|
+
* ## Why the profile is chosen, never guessed
|
|
9
|
+
*
|
|
10
|
+
* The old engine issued ONE request and treated whatever came back as the whole
|
|
11
|
+
* dataset. Against a Spring API that defaults to 20 records per page, importing
|
|
12
|
+
* a 10.000-row table in wipe-and-replace mode deleted 10.000 rows, wrote 20, and
|
|
13
|
+
* reported success. Guessing wrong is indistinguishable from a small dataset, so
|
|
14
|
+
* the profile is a deliberate answer and `none` genuinely means "one request".
|
|
15
|
+
*
|
|
16
|
+
* ## Why every stop condition is explicit
|
|
17
|
+
*
|
|
18
|
+
* A paging walk that fails to terminate is worse than one that stops early: it
|
|
19
|
+
* hammers somebody else's production API. Each profile below ends on a positive
|
|
20
|
+
* signal (a short page, an exhausted total, a missing cursor) and the whole walk
|
|
21
|
+
* is additionally bounded by {@link MAX_PAGES}.
|
|
22
|
+
*/
|
|
23
|
+
import type { PaginationProfile } from './types';
|
|
24
|
+
/**
|
|
25
|
+
* Hard ceiling on requests for one endpoint walk.
|
|
26
|
+
*
|
|
27
|
+
* Reaching it is reported as a failure rather than treated as the end of the
|
|
28
|
+
* data — a truncated read that looks complete is exactly what makes a destructive
|
|
29
|
+
* import destroy records it never saw.
|
|
30
|
+
*/
|
|
31
|
+
export declare const MAX_PAGES = 1000;
|
|
32
|
+
/** Default page size when the operator did not choose one. */
|
|
33
|
+
export declare const DEFAULT_PAGE_SIZE = 200;
|
|
34
|
+
/** How far the walk has got. Owned by the caller, advanced by {@link nextPage}. */
|
|
35
|
+
export type PageState = {
|
|
36
|
+
/** Pages already fetched. */
|
|
37
|
+
pagesFetched: number;
|
|
38
|
+
/** Records already collected across all pages. */
|
|
39
|
+
recordsFetched: number;
|
|
40
|
+
/** Cursor used for the page just fetched, for the repeat-cursor check. */
|
|
41
|
+
cursor?: string | null;
|
|
42
|
+
};
|
|
43
|
+
/** What the last response looked like, as far as paging is concerned. */
|
|
44
|
+
export type PageObservation = {
|
|
45
|
+
/** Records extracted from this page. */
|
|
46
|
+
recordCount: number;
|
|
47
|
+
/** Parsed response body, for reading total/last/cursor paths. */
|
|
48
|
+
body: unknown;
|
|
49
|
+
/** Raw `Link` header, for the RFC 8288 profile. */
|
|
50
|
+
linkHeader?: string | null;
|
|
51
|
+
};
|
|
52
|
+
/** The next request to make, or the reason the walk is over. */
|
|
53
|
+
export type NextRequest = {
|
|
54
|
+
kind: 'done';
|
|
55
|
+
reason: string;
|
|
56
|
+
} | {
|
|
57
|
+
kind: 'params';
|
|
58
|
+
params: Record<string, string>;
|
|
59
|
+
} | {
|
|
60
|
+
kind: 'url';
|
|
61
|
+
url: string;
|
|
62
|
+
} | {
|
|
63
|
+
kind: 'failed';
|
|
64
|
+
reason: string;
|
|
65
|
+
};
|
|
66
|
+
export declare const initialState: () => PageState;
|
|
67
|
+
/** Query parameters for the FIRST request of a walk. */
|
|
68
|
+
export declare function firstPageParams(profile: PaginationProfile): Record<string, string>;
|
|
69
|
+
/**
|
|
70
|
+
* The total the source claims to hold, when the profile names a path to it and
|
|
71
|
+
* the value is a non-negative number. Returns null when unknown — which is not
|
|
72
|
+
* an error, only a case where the completeness check cannot run.
|
|
73
|
+
*/
|
|
74
|
+
export declare function reportedTotal(profile: PaginationProfile, body: unknown): number | null;
|
|
75
|
+
/**
|
|
76
|
+
* Decides the next request after observing a page.
|
|
77
|
+
*
|
|
78
|
+
* The caller advances `state` (pagesFetched, recordsFetched, cursor) before the
|
|
79
|
+
* next call; this function only reads it.
|
|
80
|
+
*/
|
|
81
|
+
export declare function nextPage(profile: PaginationProfile, state: PageState, observed: PageObservation): NextRequest;
|
|
82
|
+
/**
|
|
83
|
+
* Extracts the `rel="next"` URL from an RFC 8288 `Link` header.
|
|
84
|
+
*
|
|
85
|
+
* Tolerant of the shapes seen in the wild: single or double quotes around the
|
|
86
|
+
* rel value, no quotes at all, extra parameters, and several links in one header.
|
|
87
|
+
* Returns null when there is no next link — which is how this profile ends.
|
|
88
|
+
*/
|
|
89
|
+
export declare function parseNextLink(header: string | null | undefined): string | null;
|
|
90
|
+
/**
|
|
91
|
+
* Is what we fetched everything the source says it has?
|
|
92
|
+
*
|
|
93
|
+
* Only meaningful when the source reports a total. The gate this feeds refuses a
|
|
94
|
+
* DESTRUCTIVE run on a mismatch: deleting rows the import never fetched is the
|
|
95
|
+
* one failure that cannot be undone by running it again.
|
|
96
|
+
*/
|
|
97
|
+
export declare function isComplete(fetched: number, reported: number | null): boolean;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deciding what a run would do — before it does any of it.
|
|
3
|
+
*
|
|
4
|
+
* Pure. Given the mapped records, the rows already in the target table, the write
|
|
5
|
+
* mode and the missing-record policy, this produces the exact list of creates,
|
|
6
|
+
* updates, deletes and deactivations. The transaction that follows executes this
|
|
7
|
+
* list and makes no decisions of its own.
|
|
8
|
+
*
|
|
9
|
+
* The split matters for two reasons. The preview screen and the real run share
|
|
10
|
+
* this function, so what an operator is shown is what will happen rather than a
|
|
11
|
+
* second implementation that drifts. And every rule about destroying data is
|
|
12
|
+
* testable here without a database.
|
|
13
|
+
*/
|
|
14
|
+
import type { MappedRecord, MissingRecordPolicy, RunIssue, RunPlan, WriteMode } from './types';
|
|
15
|
+
/** A row already in the target table. */
|
|
16
|
+
export type ExistingRow = {
|
|
17
|
+
id: string;
|
|
18
|
+
row: Record<string, unknown>;
|
|
19
|
+
};
|
|
20
|
+
export type CreateOp = {
|
|
21
|
+
data: Record<string, unknown>;
|
|
22
|
+
sourceIndex: number;
|
|
23
|
+
};
|
|
24
|
+
export type UpdateOp = {
|
|
25
|
+
id: string;
|
|
26
|
+
data: Record<string, unknown>;
|
|
27
|
+
before: Record<string, unknown>;
|
|
28
|
+
sourceIndex: number;
|
|
29
|
+
};
|
|
30
|
+
/** Exactly what to execute. Nothing here needs interpreting. */
|
|
31
|
+
export type WriteOperations = {
|
|
32
|
+
creates: CreateOp[];
|
|
33
|
+
updates: UpdateOp[];
|
|
34
|
+
/** Rows to remove — from `replace`, or from the `delete` missing policy. */
|
|
35
|
+
deletes: ExistingRow[];
|
|
36
|
+
/** Rows to mark inactive because the source no longer carries them. */
|
|
37
|
+
deactivates: ExistingRow[];
|
|
38
|
+
issues: RunIssue[];
|
|
39
|
+
warnings: string[];
|
|
40
|
+
};
|
|
41
|
+
export type PlanInput = {
|
|
42
|
+
records: readonly MappedRecord[];
|
|
43
|
+
/** Every row currently in the target table. */
|
|
44
|
+
existing: readonly ExistingRow[];
|
|
45
|
+
/** Target column that identifies the same record across runs, if configured. */
|
|
46
|
+
dedupTargetField: string | null;
|
|
47
|
+
writeMode: WriteMode;
|
|
48
|
+
missingPolicy: MissingRecordPolicy;
|
|
49
|
+
/** Column and value written when the policy is `deactivate`. */
|
|
50
|
+
missingColumn?: string | null;
|
|
51
|
+
missingValue?: string | null;
|
|
52
|
+
/** NOT NULL, no-default columns of the target — a row missing one cannot be written. */
|
|
53
|
+
requiredFields?: readonly string[];
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Works out every write a run would perform.
|
|
57
|
+
*
|
|
58
|
+
* Records are processed in source order so an issue's `sourceIndex` always points
|
|
59
|
+
* at the record an operator can find in the sample.
|
|
60
|
+
*/
|
|
61
|
+
export declare function planWrites(input: PlanInput): WriteOperations;
|
|
62
|
+
/** Applies the deactivation column/value to a row's update payload. */
|
|
63
|
+
export declare function deactivationPayload(missingColumn: string, missingValue: string): Record<string, unknown>;
|
|
64
|
+
/** Turns the operations into the summary a preview screen renders. */
|
|
65
|
+
export declare function summarize(ops: WriteOperations, meta: {
|
|
66
|
+
targetEntity: string;
|
|
67
|
+
writeMode: WriteMode;
|
|
68
|
+
missingPolicy: MissingRecordPolicy;
|
|
69
|
+
totalFetched: number;
|
|
70
|
+
reportedTotal: number | null;
|
|
71
|
+
existingCount: number;
|
|
72
|
+
dedupConfigured: boolean;
|
|
73
|
+
sampleSize?: number;
|
|
74
|
+
}): RunPlan;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Undoing a run that already succeeded.
|
|
3
|
+
*
|
|
4
|
+
* A failed run needs nothing undone — its transaction rolled back. This is for
|
|
5
|
+
* the other case: the import worked, and afterwards somebody noticed the data was
|
|
6
|
+
* wrong. Bad records at the source, a mapping pointed at the wrong column, the
|
|
7
|
+
* wrong endpoint. Until now there was no way back from that at all; a wipe-and-
|
|
8
|
+
* replace ran and the previous contents were simply gone.
|
|
9
|
+
*
|
|
10
|
+
* ## Reversing, not re-importing
|
|
11
|
+
*
|
|
12
|
+
* Each before-image is turned back into its own opposite: a row this run created
|
|
13
|
+
* is deleted, a row it changed is written back to what it was, a row it deleted
|
|
14
|
+
* is put back. All in one transaction, so a revert is as all-or-nothing as the
|
|
15
|
+
* run it undoes.
|
|
16
|
+
*
|
|
17
|
+
* ## Somebody else's edit is not ours to discard
|
|
18
|
+
*
|
|
19
|
+
* Between the run and the revert, a person may have edited one of these rows by
|
|
20
|
+
* hand. Writing the before-image over that would destroy their work while
|
|
21
|
+
* appearing to be a safe undo. So a row whose current state no longer matches
|
|
22
|
+
* what the run left behind is reported as a conflict and skipped — unless the
|
|
23
|
+
* operator explicitly asks to overwrite it, having been told which rows.
|
|
24
|
+
*/
|
|
25
|
+
import type { PreImage, WriterDeps } from './writer';
|
|
26
|
+
export type RevertOperations = {
|
|
27
|
+
/** Rows the run created, to be removed again. */
|
|
28
|
+
deletes: string[];
|
|
29
|
+
/** Rows to be written back to their previous contents. */
|
|
30
|
+
restores: {
|
|
31
|
+
id: string;
|
|
32
|
+
row: Record<string, unknown>;
|
|
33
|
+
}[];
|
|
34
|
+
/** Rows the run deleted, to be inserted again. */
|
|
35
|
+
reinserts: Record<string, unknown>[];
|
|
36
|
+
/** Rows changed since the run; not touched. */
|
|
37
|
+
conflicts: {
|
|
38
|
+
id: string;
|
|
39
|
+
reason: string;
|
|
40
|
+
}[];
|
|
41
|
+
};
|
|
42
|
+
export type RevertPlanInput = {
|
|
43
|
+
preImages: readonly PreImage[];
|
|
44
|
+
/** Current contents of every row the run touched, by id. Absent = already gone. */
|
|
45
|
+
current: ReadonlyMap<string, Record<string, unknown>>;
|
|
46
|
+
/** Overwrite rows that changed since the run, having reported them first. */
|
|
47
|
+
force?: boolean;
|
|
48
|
+
/** Columns excluded from the changed-since check — they move on every write. */
|
|
49
|
+
ignoreColumns?: readonly string[];
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Works out how to undo a run. Pure — every rule about whose data may be
|
|
53
|
+
* overwritten is decided here and testable without a database.
|
|
54
|
+
*/
|
|
55
|
+
export declare function planRevert(input: RevertPlanInput): RevertOperations;
|
|
56
|
+
export type RevertResult = {
|
|
57
|
+
ok: true;
|
|
58
|
+
deleted: number;
|
|
59
|
+
restored: number;
|
|
60
|
+
reinserted: number;
|
|
61
|
+
conflicts: number;
|
|
62
|
+
} | {
|
|
63
|
+
ok: false;
|
|
64
|
+
error: string;
|
|
65
|
+
conflicts: number;
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Applies a revert plan in one transaction.
|
|
69
|
+
*
|
|
70
|
+
* Order mirrors the run it undoes: removals first, so an identifier or unique
|
|
71
|
+
* value freed here is available to the re-inserts that follow.
|
|
72
|
+
*/
|
|
73
|
+
export declare function applyRevert(deps: WriterDeps, ops: RevertOperations, meta: {
|
|
74
|
+
targetEntity: string;
|
|
75
|
+
userId: string;
|
|
76
|
+
}): Promise<RevertResult>;
|
|
77
|
+
/** Ids a caller must read the current state of before planning a revert. */
|
|
78
|
+
export declare function touchedRowIds(preImages: readonly PreImage[]): string[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running an import, start to finish.
|
|
3
|
+
*
|
|
4
|
+
* The order below is the whole design. Everything is fetched, mapped, planned and
|
|
5
|
+
* validated before a single row is written, and the write itself is one
|
|
6
|
+
* transaction. There is no state in which a run has half-happened.
|
|
7
|
+
*
|
|
8
|
+
* ```
|
|
9
|
+
* lock one run per mapping, across replicas
|
|
10
|
+
* collect every endpoint, in dependency order, all pages
|
|
11
|
+
* completeness refuse a destructive run that did not read everything
|
|
12
|
+
* map external records → our columns
|
|
13
|
+
* plan creates / updates / deletes / deactivations
|
|
14
|
+
* apply ONE transaction; any failure unwinds it
|
|
15
|
+
* record before-images, counts, and the run row
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* Every step reports progress and checks for cancellation, because a chained
|
|
19
|
+
* import can legitimately take minutes and an operator who changes their mind
|
|
20
|
+
* must be able to stop it — not merely close the tab and hope.
|
|
21
|
+
*/
|
|
22
|
+
import type { Logger } from '../Logger';
|
|
23
|
+
import { type EndpointSpec } from './collect';
|
|
24
|
+
import type { FetchImpl, SourceConnection } from './fetcher';
|
|
25
|
+
import { type OAuthConfig, type TokenCache } from './oauth';
|
|
26
|
+
import { type ExistingRow } from './plan';
|
|
27
|
+
import type { FieldMapping, MissingRecordPolicy, RunFailure, RunIssue, RunPlan, RunProgress, WriteMode } from './types';
|
|
28
|
+
import { type PreImage, type WriterDeps } from './writer';
|
|
29
|
+
/** The mapping being run, resolved from the database by the caller. */
|
|
30
|
+
export type MappingSpec = {
|
|
31
|
+
id: string;
|
|
32
|
+
name: string;
|
|
33
|
+
targetEntity: string;
|
|
34
|
+
fieldMappings: FieldMapping[];
|
|
35
|
+
dedupTargetField: string | null;
|
|
36
|
+
writeMode: WriteMode;
|
|
37
|
+
missingPolicy: MissingRecordPolicy;
|
|
38
|
+
missingColumn?: string | null;
|
|
39
|
+
missingValue?: string | null;
|
|
40
|
+
};
|
|
41
|
+
/** Everything a run needs, already read and decrypted. */
|
|
42
|
+
export type RunContext = {
|
|
43
|
+
mapping: MappingSpec;
|
|
44
|
+
/** The endpoint this mapping reads, plus every endpoint it depends on. */
|
|
45
|
+
endpoints: EndpointSpec[];
|
|
46
|
+
/** Which of `endpoints` the mapping itself maps from. */
|
|
47
|
+
rootEndpointId: string;
|
|
48
|
+
source: SourceConnection;
|
|
49
|
+
/** Present only when the source authenticates with OAuth2. */
|
|
50
|
+
oauth?: OAuthConfig | null;
|
|
51
|
+
userId: string;
|
|
52
|
+
trigger: 'manual' | 'schedule' | 'api';
|
|
53
|
+
};
|
|
54
|
+
/** Persistence and lookups the runner needs. Injected so it can be tested. */
|
|
55
|
+
export type RunRepository = {
|
|
56
|
+
openRun(input: {
|
|
57
|
+
mappingId: string;
|
|
58
|
+
targetEntity: string;
|
|
59
|
+
writeMode: WriteMode;
|
|
60
|
+
trigger: string;
|
|
61
|
+
userId: string;
|
|
62
|
+
}): Promise<{
|
|
63
|
+
ok: true;
|
|
64
|
+
runId: string;
|
|
65
|
+
} | {
|
|
66
|
+
ok: false;
|
|
67
|
+
error: string;
|
|
68
|
+
}>;
|
|
69
|
+
closeRun(runId: string, patch: Record<string, unknown>): Promise<{
|
|
70
|
+
ok: true;
|
|
71
|
+
} | {
|
|
72
|
+
ok: false;
|
|
73
|
+
error: string;
|
|
74
|
+
}>;
|
|
75
|
+
savePreImages(runId: string, images: PreImage[]): Promise<void>;
|
|
76
|
+
/** True once a cancel request has been recorded for this run. */
|
|
77
|
+
isCancelRequested(runId: string): Promise<boolean>;
|
|
78
|
+
/** Every row currently in the target table. */
|
|
79
|
+
readExistingRows(targetEntity: string): Promise<ExistingRow[]>;
|
|
80
|
+
/** Distinct values of one column, for an `entity` parameter binding. */
|
|
81
|
+
readEntityColumn(entity: string, field: string): Promise<unknown[]>;
|
|
82
|
+
/** NOT NULL, no-default columns of the target. */
|
|
83
|
+
requiredFields(targetEntity: string): string[];
|
|
84
|
+
};
|
|
85
|
+
export type RunnerDeps = {
|
|
86
|
+
repo: RunRepository;
|
|
87
|
+
writer: WriterDeps;
|
|
88
|
+
logger: Logger;
|
|
89
|
+
tokenCache: TokenCache;
|
|
90
|
+
fetchImpl?: FetchImpl;
|
|
91
|
+
publishProgress?: (progress: RunProgress) => void;
|
|
92
|
+
/**
|
|
93
|
+
* Called once the run record exists, before any work starts.
|
|
94
|
+
*
|
|
95
|
+
* A caller that answers 202 needs the identifier immediately and must not wait
|
|
96
|
+
* for the import. Racing a timer against the run would be a guess; this is the
|
|
97
|
+
* moment the identifier actually becomes real.
|
|
98
|
+
*/
|
|
99
|
+
onRunOpened?: (runId: string) => void;
|
|
100
|
+
maxRecordsPerRun?: number;
|
|
101
|
+
maxConcurrentFanout?: number;
|
|
102
|
+
retainPreImagesMs?: number;
|
|
103
|
+
/** External cancellation, e.g. the process shutting down. */
|
|
104
|
+
abortSignal?: AbortSignal;
|
|
105
|
+
};
|
|
106
|
+
export type RunOutcome = {
|
|
107
|
+
runId: string | null;
|
|
108
|
+
status: 'succeeded' | 'failed' | 'cancelled';
|
|
109
|
+
failure: RunFailure | null;
|
|
110
|
+
plan: RunPlan | null;
|
|
111
|
+
created: number;
|
|
112
|
+
updated: number;
|
|
113
|
+
deleted: number;
|
|
114
|
+
deactivated: number;
|
|
115
|
+
externalCalls: number;
|
|
116
|
+
issues: RunIssue[];
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Fetches everything, decides everything, then writes it once.
|
|
120
|
+
*
|
|
121
|
+
* Returns rather than throws: a failed import is an outcome an operator reads,
|
|
122
|
+
* not an exception for a route handler to translate.
|
|
123
|
+
*/
|
|
124
|
+
export declare function executeRun(context: RunContext, deps: RunnerDeps, lock: {
|
|
125
|
+
isHeld: () => boolean;
|
|
126
|
+
}): Promise<RunOutcome>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|