nucleus-core-ts 0.9.781 → 0.9.783

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/client.js +1 -1
  2. package/dist/fe/components/IntegrationsPage/components/ConnectionAuthFields.d.ts +2 -0
  3. package/dist/fe/components/IntegrationsPage/components/ConnectionAuthFields.js +85 -0
  4. package/dist/fe/components/IntegrationsPage/components/ConnectionBasicFields.d.ts +2 -0
  5. package/dist/fe/components/IntegrationsPage/components/ConnectionBasicFields.js +165 -0
  6. package/dist/fe/components/IntegrationsPage/components/ConnectionForm.d.ts +10 -0
  7. package/dist/fe/components/IntegrationsPage/components/ConnectionForm.js +112 -0
  8. package/dist/fe/components/IntegrationsPage/components/ConnectionList.d.ts +9 -0
  9. package/dist/fe/components/IntegrationsPage/components/ConnectionList.js +184 -0
  10. package/dist/fe/components/IntegrationsPage/components/ConnectionOauthFields.d.ts +2 -0
  11. package/dist/fe/components/IntegrationsPage/components/ConnectionOauthFields.js +131 -0
  12. package/dist/fe/components/IntegrationsPage/components/ConnectionSettingsTab.d.ts +21 -0
  13. package/dist/fe/components/IntegrationsPage/components/ConnectionSettingsTab.js +49 -0
  14. package/dist/fe/components/IntegrationsPage/components/ConnectionTabs.d.ts +12 -0
  15. package/dist/fe/components/IntegrationsPage/components/ConnectionTabs.js +95 -0
  16. package/dist/fe/components/IntegrationsPage/components/ConnectionTest.d.ts +9 -0
  17. package/dist/fe/components/IntegrationsPage/components/ConnectionTest.js +116 -0
  18. package/dist/fe/components/IntegrationsPage/components/CredentialFields.d.ts +9 -0
  19. package/dist/fe/components/IntegrationsPage/components/CredentialFields.js +171 -0
  20. package/dist/fe/components/IntegrationsPage/components/CredentialSlotRow.d.ts +13 -0
  21. package/dist/fe/components/IntegrationsPage/components/CredentialSlotRow.js +63 -0
  22. package/dist/fe/components/IntegrationsPage/components/EndpointForm.d.ts +11 -0
  23. package/dist/fe/components/IntegrationsPage/components/EndpointForm.js +166 -0
  24. package/dist/fe/components/IntegrationsPage/components/EndpointFormFields.d.ts +19 -0
  25. package/dist/fe/components/IntegrationsPage/components/EndpointFormFields.js +162 -0
  26. package/dist/fe/components/IntegrationsPage/components/EndpointList.d.ts +12 -0
  27. package/dist/fe/components/IntegrationsPage/components/EndpointList.js +205 -0
  28. package/dist/fe/components/IntegrationsPage/components/EndpointSample.d.ts +6 -0
  29. package/dist/fe/components/IntegrationsPage/components/EndpointSample.js +204 -0
  30. package/dist/fe/components/IntegrationsPage/components/EndpointsTab.d.ts +5 -0
  31. package/dist/fe/components/IntegrationsPage/components/EndpointsTab.js +101 -0
  32. package/dist/fe/components/IntegrationsPage/components/FieldMappingRow.d.ts +17 -0
  33. package/dist/fe/components/IntegrationsPage/components/FieldMappingRow.js +179 -0
  34. package/dist/fe/components/IntegrationsPage/components/FieldMappingRows.d.ts +11 -0
  35. package/dist/fe/components/IntegrationsPage/components/FieldMappingRows.js +115 -0
  36. package/dist/fe/components/IntegrationsPage/components/IntegrationsPage.d.ts +2 -0
  37. package/dist/fe/components/IntegrationsPage/components/IntegrationsPage.js +175 -0
  38. package/dist/fe/components/IntegrationsPage/components/MappingEditor.d.ts +15 -0
  39. package/dist/fe/components/IntegrationsPage/components/MappingEditor.js +193 -0
  40. package/dist/fe/components/IntegrationsPage/components/MappingList.d.ts +11 -0
  41. package/dist/fe/components/IntegrationsPage/components/MappingList.js +191 -0
  42. package/dist/fe/components/IntegrationsPage/components/MappingWriteModeFields.d.ts +19 -0
  43. package/dist/fe/components/IntegrationsPage/components/MappingWriteModeFields.js +150 -0
  44. package/dist/fe/components/IntegrationsPage/components/MappingsTab.d.ts +5 -0
  45. package/dist/fe/components/IntegrationsPage/components/MappingsTab.js +154 -0
  46. package/dist/fe/components/IntegrationsPage/components/PaginationFields.d.ts +9 -0
  47. package/dist/fe/components/IntegrationsPage/components/PaginationFields.js +218 -0
  48. package/dist/fe/components/IntegrationsPage/components/ParamBindingRow.d.ts +24 -0
  49. package/dist/fe/components/IntegrationsPage/components/ParamBindingRow.js +180 -0
  50. package/dist/fe/components/IntegrationsPage/components/ParamBindingRows.d.ts +33 -0
  51. package/dist/fe/components/IntegrationsPage/components/ParamBindingRows.js +100 -0
  52. package/dist/fe/components/IntegrationsPage/components/Primitives.d.ts +65 -0
  53. package/dist/fe/components/IntegrationsPage/components/Primitives.js +94 -0
  54. package/dist/fe/components/IntegrationsPage/components/RunConfirmStep.d.ts +10 -0
  55. package/dist/fe/components/IntegrationsPage/components/RunConfirmStep.js +48 -0
  56. package/dist/fe/components/IntegrationsPage/components/RunHistory.d.ts +10 -0
  57. package/dist/fe/components/IntegrationsPage/components/RunHistory.js +124 -0
  58. package/dist/fe/components/IntegrationsPage/components/RunHistoryRow.d.ts +7 -0
  59. package/dist/fe/components/IntegrationsPage/components/RunHistoryRow.js +129 -0
  60. package/dist/fe/components/IntegrationsPage/components/RunPanel.d.ts +9 -0
  61. package/dist/fe/components/IntegrationsPage/components/RunPanel.js +160 -0
  62. package/dist/fe/components/IntegrationsPage/components/RunPlanIssues.d.ts +5 -0
  63. package/dist/fe/components/IntegrationsPage/components/RunPlanIssues.js +43 -0
  64. package/dist/fe/components/IntegrationsPage/components/RunPlanSummary.d.ts +5 -0
  65. package/dist/fe/components/IntegrationsPage/components/RunPlanSummary.js +92 -0
  66. package/dist/fe/components/IntegrationsPage/components/RunProgressBar.d.ts +8 -0
  67. package/dist/fe/components/IntegrationsPage/components/RunProgressBar.js +77 -0
  68. package/dist/fe/components/IntegrationsPage/components/RunRevertConflicts.d.ts +13 -0
  69. package/dist/fe/components/IntegrationsPage/components/RunRevertConflicts.js +65 -0
  70. package/dist/fe/components/IntegrationsPage/components/RunsTab.d.ts +13 -0
  71. package/dist/fe/components/IntegrationsPage/components/RunsTab.js +39 -0
  72. package/dist/fe/components/IntegrationsPage/components/Skeletons.d.ts +9 -0
  73. package/dist/fe/components/IntegrationsPage/components/Skeletons.js +139 -0
  74. package/dist/fe/components/IntegrationsPage/components/authOptions.d.ts +29 -0
  75. package/dist/fe/components/IntegrationsPage/components/authOptions.js +71 -0
  76. package/dist/fe/components/IntegrationsPage/components/connectionDraft.d.ts +64 -0
  77. package/dist/fe/components/IntegrationsPage/components/connectionDraft.js +112 -0
  78. package/dist/fe/components/IntegrationsPage/components/credentialSlots.d.ts +25 -0
  79. package/dist/fe/components/IntegrationsPage/components/credentialSlots.js +67 -0
  80. package/dist/fe/components/IntegrationsPage/components/fieldMappingOptions.d.ts +16 -0
  81. package/dist/fe/components/IntegrationsPage/components/fieldMappingOptions.js +41 -0
  82. package/dist/fe/components/IntegrationsPage/components/formSupport.d.ts +19 -0
  83. package/dist/fe/components/IntegrationsPage/components/formSupport.js +25 -0
  84. package/dist/fe/components/IntegrationsPage/components/mappingDraft.d.ts +40 -0
  85. package/dist/fe/components/IntegrationsPage/components/mappingDraft.js +44 -0
  86. package/dist/fe/components/IntegrationsPage/components/pathParams.d.ts +9 -0
  87. package/dist/fe/components/IntegrationsPage/components/pathParams.js +15 -0
  88. package/dist/fe/components/IntegrationsPage/components/runFormat.d.ts +33 -0
  89. package/dist/fe/components/IntegrationsPage/components/runFormat.js +85 -0
  90. package/dist/fe/components/IntegrationsPage/components/useInfiniteList.d.ts +40 -0
  91. package/dist/fe/components/IntegrationsPage/components/useInfiniteList.js +110 -0
  92. package/dist/fe/components/IntegrationsPage/index.d.ts +18 -0
  93. package/dist/fe/components/IntegrationsPage/index.js +11 -0
  94. package/dist/fe/components/IntegrationsPage/store/index.d.ts +2 -0
  95. package/dist/fe/components/IntegrationsPage/store/index.js +124 -0
  96. package/dist/fe/components/IntegrationsPage/store/state.d.ts +80 -0
  97. package/dist/fe/components/IntegrationsPage/store/state.js +32 -0
  98. package/dist/fe/components/IntegrationsPage/theme/index.d.ts +103 -0
  99. package/dist/fe/components/IntegrationsPage/theme/index.js +113 -0
  100. package/dist/fe/components/IntegrationsPage/types/actions.d.ts +91 -0
  101. package/dist/fe/components/IntegrationsPage/types/actions.js +1 -0
  102. package/dist/fe/components/IntegrationsPage/types/index.d.ts +3 -0
  103. package/dist/fe/components/IntegrationsPage/types/index.js +3 -0
  104. package/dist/fe/components/IntegrationsPage/types/records.d.ts +139 -0
  105. package/dist/fe/components/IntegrationsPage/types/records.js +11 -0
  106. package/dist/fe/components/IntegrationsPage/types/results.d.ts +52 -0
  107. package/dist/fe/components/IntegrationsPage/types/results.js +4 -0
  108. package/dist/fe/index.d.ts +2 -0
  109. package/dist/fe/index.js +1 -0
  110. package/dist/index.js +14 -14
  111. package/dist/src/Client/ApiCaller/system-tables.d.ts +505 -0
  112. package/dist/src/Client/Proxy/wsProxy.js +25 -1
  113. package/dist/src/ElysiaPlugin/routes/backup/types.d.ts +3 -1
  114. package/dist/src/ElysiaPlugin/routes/integrations/index.d.ts +49 -0
  115. package/dist/src/ElysiaPlugin/routes/integrations/inspect.d.ts +87 -0
  116. package/dist/src/ElysiaPlugin/routes/integrations/inspectRoutes.d.ts +41 -0
  117. package/dist/src/ElysiaPlugin/routes/integrations/repository.d.ts +88 -0
  118. package/dist/src/ElysiaPlugin/routes/integrations/types.d.ts +22 -0
  119. package/dist/src/ElysiaPlugin/routes/pubsub/frame.d.ts +37 -0
  120. package/dist/src/ElysiaPlugin/routes/pubsub/frame.test.d.ts +1 -0
  121. package/dist/src/Services/Backup/BackupService.d.ts +16 -8
  122. package/dist/src/Services/Backup/types.d.ts +25 -0
  123. package/dist/src/Services/Integrations/bindings.d.ts +116 -0
  124. package/dist/src/Services/Integrations/bindings.test.d.ts +1 -0
  125. package/dist/src/Services/Integrations/collect.d.ts +68 -0
  126. package/dist/src/Services/Integrations/collect.test.d.ts +1 -0
  127. package/dist/src/Services/Integrations/fetcher.d.ts +100 -0
  128. package/dist/src/Services/Integrations/fetcher.test.d.ts +1 -0
  129. package/dist/src/Services/Integrations/lock.d.ts +77 -0
  130. package/dist/src/Services/Integrations/lock.test.d.ts +1 -0
  131. package/dist/src/Services/Integrations/mapping.d.ts +63 -0
  132. package/dist/src/Services/Integrations/mapping.test.d.ts +1 -0
  133. package/dist/src/Services/Integrations/oauth.d.ts +89 -0
  134. package/dist/src/Services/Integrations/oauth.test.d.ts +1 -0
  135. package/dist/src/Services/Integrations/pagination.d.ts +97 -0
  136. package/dist/src/Services/Integrations/pagination.test.d.ts +1 -0
  137. package/dist/src/Services/Integrations/plan.d.ts +74 -0
  138. package/dist/src/Services/Integrations/plan.test.d.ts +1 -0
  139. package/dist/src/Services/Integrations/revert.d.ts +78 -0
  140. package/dist/src/Services/Integrations/revert.test.d.ts +1 -0
  141. package/dist/src/Services/Integrations/runner.d.ts +126 -0
  142. package/dist/src/Services/Integrations/runner.test.d.ts +1 -0
  143. package/dist/src/Services/Integrations/types.d.ts +208 -0
  144. package/dist/src/Services/Integrations/writer.d.ts +105 -0
  145. package/dist/src/Services/Integrations/writer.test.d.ts +1 -0
  146. package/dist/src/Utils/cron.d.ts +65 -0
  147. package/dist/src/Utils/cron.test.d.ts +1 -0
  148. package/dist/src/types.d.ts +67 -1
  149. package/package.json +1 -1
  150. package/schemas/config.nucleus.json +43 -0
  151. package/schemas/table.schema.json +2 -1
  152. package/scripts/generate-schema.test.ts +78 -0
  153. package/scripts/generate-schema.ts +43 -5
  154. package/src/system.tables.json +362 -0
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Making sure one mapping imports at a time, across every replica.
3
+ *
4
+ * Two runs of the same mapping overlapping is not a performance problem, it is a
5
+ * correctness one: both read the same "what is already here", both decide the
6
+ * same rows are missing, and both act on that decision. Under a `delete` policy
7
+ * that removes rows the second run has already re-created.
8
+ *
9
+ * ## Why not `RedisManager.acquireLock` directly
10
+ *
11
+ * That helper stores a literal `'1'`, so `releaseLock` cannot tell whose lock it
12
+ * is deleting. If run A stalls long enough for its lock to expire and run B takes
13
+ * it, A finishing normally then deletes B's lock and a third run walks straight
14
+ * in. The holder here writes a token and checks it before releasing, so a lock it
15
+ * no longer owns is left alone.
16
+ *
17
+ * ## Why it renews
18
+ *
19
+ * A fixed TTL has to be either longer than the slowest import — in which case a
20
+ * crashed pod blocks the mapping for that long — or shorter, in which case a
21
+ * healthy long import loses its lock halfway. Renewing on a timer gives both: the
22
+ * TTL stays short, and a holder that dies simply stops renewing.
23
+ */
24
+ import type { Logger } from '../Logger';
25
+ type RedisResult<T> = {
26
+ success: true;
27
+ data: T;
28
+ } | {
29
+ success: false;
30
+ error: string;
31
+ };
32
+ /** The slice of the Redis manager this needs. */
33
+ export type LockStore = {
34
+ create<T>(key: string, value: T, ttlSeconds?: number): Promise<RedisResult<'OK'>>;
35
+ read<T>(key: string): Promise<RedisResult<T | null>>;
36
+ update<T>(key: string, value: T, preserveTtl?: boolean): Promise<RedisResult<'OK'>>;
37
+ remove(key: string): Promise<RedisResult<number>>;
38
+ exists(key: string): Promise<RedisResult<boolean>>;
39
+ };
40
+ export type LockHandle = {
41
+ /** Stops renewing and releases, but only if this holder still owns the lock. */
42
+ release: () => Promise<void>;
43
+ /** False once a renewal has found the lock owned by somebody else. */
44
+ isHeld: () => boolean;
45
+ token: string;
46
+ };
47
+ export type AcquireResult = {
48
+ ok: true;
49
+ handle: LockHandle;
50
+ } | {
51
+ ok: false;
52
+ reason: 'held';
53
+ heldSince: number | null;
54
+ } | {
55
+ ok: false;
56
+ reason: 'unavailable';
57
+ error: string;
58
+ };
59
+ /** Short by design — a dead holder should block the mapping for seconds, not hours. */
60
+ export declare const LOCK_TTL_SECONDS = 60;
61
+ /** Renew well inside the TTL so one slow renewal does not lose the lock. */
62
+ export declare const RENEW_INTERVAL_MS = 20000;
63
+ export declare const lockKeyFor: (mappingId: string) => string;
64
+ /**
65
+ * Takes the lock for a mapping, or reports who holds it.
66
+ *
67
+ * Redis being unreachable is reported as `unavailable`, NOT as "free". Treating a
68
+ * broken lock service as an open door is how two runs end up deleting each
69
+ * other's records — the caller decides whether to proceed, and for a destructive
70
+ * write mode the answer should be no.
71
+ */
72
+ export declare function acquireRunLock(store: LockStore, mappingId: string, deps?: {
73
+ logger?: Logger;
74
+ now?: () => number;
75
+ ttlSeconds?: number;
76
+ }): Promise<AcquireResult>;
77
+ export {};
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,63 @@
1
+ /**
2
+ * Turning a third-party record into one of our rows. Pure — no I/O, no database,
3
+ * no config. Everything here is a function of its arguments, so the rules that
4
+ * decide what lands in a column can be tested without a server or a source.
5
+ */
6
+ import type { FieldMapping, MappedRecord, MappingTransform, RunIssue } from './types';
7
+ /** Reads a possibly-nested value by dot-path (`a.b.c`). */
8
+ export declare function getByPath(obj: unknown, path: string): unknown;
9
+ /**
10
+ * Pulls the record array out of a response body.
11
+ *
12
+ * `rootPath` always wins. Without one we look for the usual envelope keys, then
13
+ * fall back to a bounded search for the first array of objects anywhere in the
14
+ * body — plenty of APIs wrap their list under a name nobody would guess.
15
+ *
16
+ * A rootPath that points at a single object yields that object as one record,
17
+ * because "fetch this one thing" is a legitimate endpoint.
18
+ *
19
+ * Never throws. An unrecognisable body yields no records, and the caller decides
20
+ * whether that is an error — for a root endpoint it is; for a child call in a
21
+ * fan-out it may just mean this particular parent has none.
22
+ */
23
+ export declare function extractRecords(body: unknown, rootPath?: string | null): unknown[];
24
+ /**
25
+ * The outcome of one transform.
26
+ *
27
+ * A failed conversion is reported rather than silently becoming null. "The HR
28
+ * system sent `N/A` where a number was expected" and "the field was empty" are
29
+ * different facts, and only the first one is worth an operator's attention — the
30
+ * old engine collapsed both to null and neither reached anyone.
31
+ */
32
+ export type TransformResult = {
33
+ ok: true;
34
+ value: unknown;
35
+ } | {
36
+ ok: false;
37
+ reason: string;
38
+ raw: unknown;
39
+ };
40
+ /** Applies one transform. `none`/unknown returns the value untouched. */
41
+ export declare function applyTransform(value: unknown, transform?: MappingTransform): TransformResult;
42
+ /**
43
+ * Maps ONE external record onto a row.
44
+ *
45
+ * `parent`, when present, is exposed to mappings under the `__parent` prefix so a
46
+ * child record fetched per-parent can carry its parent's fields — e.g. the company
47
+ * name that produced this employee. Without it a fan-out loses the very thing that
48
+ * distinguishes one batch from the next.
49
+ */
50
+ export declare function mapRecord(external: unknown, mappings: readonly FieldMapping[], parent?: unknown): {
51
+ row: Record<string, unknown>;
52
+ problems: string[];
53
+ };
54
+ /** Maps a batch, carrying dedup keys and per-record problems out with it. */
55
+ export declare function buildMappedRecords(externalRecords: readonly unknown[], mappings: readonly FieldMapping[], dedupTargetField?: string | null, parents?: readonly unknown[]): {
56
+ records: MappedRecord[];
57
+ issues: RunIssue[];
58
+ };
59
+ /**
60
+ * Dot-paths of a record's leaf fields, for the mapping screen's field picker.
61
+ * Capped in both breadth and depth — this feeds a dropdown, not an analysis.
62
+ */
63
+ export declare function flattenKeys(obj: unknown, prefix?: string, out?: string[], depth?: number): string[];
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Minting and reusing OAuth2 bearer tokens for a source.
3
+ *
4
+ * Supports the two grants a server-to-server integration actually uses:
5
+ * `client_credentials`, and the resource-owner `password` grant that Keycloak
6
+ * deployments still hand out for service accounts. An access token is short-lived
7
+ * by design, so it is cached and renewed rather than re-minted per request.
8
+ *
9
+ * ## The cache key includes the credentials
10
+ *
11
+ * Keying only by source id — which is what the engine this replaces did — means
12
+ * that after an operator corrects a wrong password, the cached token minted with
13
+ * the OLD one keeps being used until it expires. The integration goes on failing
14
+ * for up to an hour after it was fixed, and nothing on screen explains why.
15
+ * Hashing the credentials into the key makes a corrected credential a cache miss
16
+ * immediately.
17
+ *
18
+ * The hash is one-way: the cache key never holds the secret itself.
19
+ */
20
+ import { type FetchImpl } from './fetcher';
21
+ export type OAuthGrant = 'password' | 'client_credentials';
22
+ export type OAuthConfig = {
23
+ sourceId: string;
24
+ tokenUrl: string;
25
+ grantType: OAuthGrant;
26
+ clientId?: string | null;
27
+ /** Decrypted. */
28
+ clientSecret?: string | null;
29
+ username?: string | null;
30
+ /** Decrypted. */
31
+ password?: string | null;
32
+ scope?: string | null;
33
+ /** How client credentials are presented: an HTTP Basic header, or form fields. */
34
+ clientAuth?: 'basic' | 'body';
35
+ tlsVerify?: boolean;
36
+ timeoutMs?: number;
37
+ };
38
+ export type CachedToken = {
39
+ accessToken: string;
40
+ expiresAt: number;
41
+ refreshToken?: string;
42
+ refreshExpiresAt?: number;
43
+ /** Fingerprint of the credentials this token was minted with. */
44
+ fingerprint: string;
45
+ };
46
+ export type TokenResult = {
47
+ ok: true;
48
+ accessToken: string;
49
+ fromCache: boolean;
50
+ } | {
51
+ ok: false;
52
+ error: string;
53
+ };
54
+ /**
55
+ * One-way fingerprint of everything that changes which token a config produces.
56
+ *
57
+ * The token URL is included: pointing a source at a different realm must not
58
+ * reuse the previous realm's token.
59
+ */
60
+ export declare function credentialFingerprint(config: OAuthConfig): string;
61
+ /** In-memory token cache. Per process — a token is not worth sharing across pods. */
62
+ export declare class TokenCache {
63
+ private readonly entries;
64
+ /**
65
+ * The stored token for a source, or null when there is none or the credentials
66
+ * have changed since it was minted. Expiry is NOT checked here — the caller
67
+ * distinguishes "expired but refreshable" from "unusable", and this cannot.
68
+ */
69
+ get(sourceId: string, fingerprint: string): CachedToken | null;
70
+ set(sourceId: string, token: CachedToken): void;
71
+ /** Drops a source's token — used by "test connection", which must mint fresh. */
72
+ invalidate(sourceId: string): void;
73
+ clear(): void;
74
+ }
75
+ /** Builds the form body and headers for one token request. */
76
+ export declare function buildTokenRequest(config: OAuthConfig, form: URLSearchParams): {
77
+ headers: Record<string, string>;
78
+ body: string;
79
+ };
80
+ /**
81
+ * Returns a usable bearer for a source.
82
+ *
83
+ * Order: a cached token that is still valid → a refresh exchange (which does not
84
+ * re-send the service account's password) → a full grant.
85
+ */
86
+ export declare function getAccessToken(config: OAuthConfig, cache: TokenCache, deps?: {
87
+ fetchImpl?: FetchImpl;
88
+ now?: () => number;
89
+ }): Promise<TokenResult>;
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,97 @@
1
+ /**
2
+ * Walking a paged endpoint.
3
+ *
4
+ * Pure: given the profile an operator chose, the walk so far, and what the last
5
+ * response contained, this decides whether to ask for another page and how. No
6
+ * fetching happens here.
7
+ *
8
+ * ## Why the profile is chosen, never guessed
9
+ *
10
+ * The old engine issued ONE request and treated whatever came back as the whole
11
+ * dataset. Against a Spring API that defaults to 20 records per page, importing
12
+ * a 10.000-row table in wipe-and-replace mode deleted 10.000 rows, wrote 20, and
13
+ * reported success. Guessing wrong is indistinguishable from a small dataset, so
14
+ * the profile is a deliberate answer and `none` genuinely means "one request".
15
+ *
16
+ * ## Why every stop condition is explicit
17
+ *
18
+ * A paging walk that fails to terminate is worse than one that stops early: it
19
+ * hammers somebody else's production API. Each profile below ends on a positive
20
+ * signal (a short page, an exhausted total, a missing cursor) and the whole walk
21
+ * is additionally bounded by {@link MAX_PAGES}.
22
+ */
23
+ import type { PaginationProfile } from './types';
24
+ /**
25
+ * Hard ceiling on requests for one endpoint walk.
26
+ *
27
+ * Reaching it is reported as a failure rather than treated as the end of the
28
+ * data — a truncated read that looks complete is exactly what makes a destructive
29
+ * import destroy records it never saw.
30
+ */
31
+ export declare const MAX_PAGES = 1000;
32
+ /** Default page size when the operator did not choose one. */
33
+ export declare const DEFAULT_PAGE_SIZE = 200;
34
+ /** How far the walk has got. Owned by the caller, advanced by {@link nextPage}. */
35
+ export type PageState = {
36
+ /** Pages already fetched. */
37
+ pagesFetched: number;
38
+ /** Records already collected across all pages. */
39
+ recordsFetched: number;
40
+ /** Cursor used for the page just fetched, for the repeat-cursor check. */
41
+ cursor?: string | null;
42
+ };
43
+ /** What the last response looked like, as far as paging is concerned. */
44
+ export type PageObservation = {
45
+ /** Records extracted from this page. */
46
+ recordCount: number;
47
+ /** Parsed response body, for reading total/last/cursor paths. */
48
+ body: unknown;
49
+ /** Raw `Link` header, for the RFC 8288 profile. */
50
+ linkHeader?: string | null;
51
+ };
52
+ /** The next request to make, or the reason the walk is over. */
53
+ export type NextRequest = {
54
+ kind: 'done';
55
+ reason: string;
56
+ } | {
57
+ kind: 'params';
58
+ params: Record<string, string>;
59
+ } | {
60
+ kind: 'url';
61
+ url: string;
62
+ } | {
63
+ kind: 'failed';
64
+ reason: string;
65
+ };
66
+ export declare const initialState: () => PageState;
67
+ /** Query parameters for the FIRST request of a walk. */
68
+ export declare function firstPageParams(profile: PaginationProfile): Record<string, string>;
69
+ /**
70
+ * The total the source claims to hold, when the profile names a path to it and
71
+ * the value is a non-negative number. Returns null when unknown — which is not
72
+ * an error, only a case where the completeness check cannot run.
73
+ */
74
+ export declare function reportedTotal(profile: PaginationProfile, body: unknown): number | null;
75
+ /**
76
+ * Decides the next request after observing a page.
77
+ *
78
+ * The caller advances `state` (pagesFetched, recordsFetched, cursor) before the
79
+ * next call; this function only reads it.
80
+ */
81
+ export declare function nextPage(profile: PaginationProfile, state: PageState, observed: PageObservation): NextRequest;
82
+ /**
83
+ * Extracts the `rel="next"` URL from an RFC 8288 `Link` header.
84
+ *
85
+ * Tolerant of the shapes seen in the wild: single or double quotes around the
86
+ * rel value, no quotes at all, extra parameters, and several links in one header.
87
+ * Returns null when there is no next link — which is how this profile ends.
88
+ */
89
+ export declare function parseNextLink(header: string | null | undefined): string | null;
90
+ /**
91
+ * Is what we fetched everything the source says it has?
92
+ *
93
+ * Only meaningful when the source reports a total. The gate this feeds refuses a
94
+ * DESTRUCTIVE run on a mismatch: deleting rows the import never fetched is the
95
+ * one failure that cannot be undone by running it again.
96
+ */
97
+ export declare function isComplete(fetched: number, reported: number | null): boolean;
@@ -0,0 +1,74 @@
1
+ /**
2
+ * Deciding what a run would do — before it does any of it.
3
+ *
4
+ * Pure. Given the mapped records, the rows already in the target table, the write
5
+ * mode and the missing-record policy, this produces the exact list of creates,
6
+ * updates, deletes and deactivations. The transaction that follows executes this
7
+ * list and makes no decisions of its own.
8
+ *
9
+ * The split matters for two reasons. The preview screen and the real run share
10
+ * this function, so what an operator is shown is what will happen rather than a
11
+ * second implementation that drifts. And every rule about destroying data is
12
+ * testable here without a database.
13
+ */
14
+ import type { MappedRecord, MissingRecordPolicy, RunIssue, RunPlan, WriteMode } from './types';
15
+ /** A row already in the target table. */
16
+ export type ExistingRow = {
17
+ id: string;
18
+ row: Record<string, unknown>;
19
+ };
20
+ export type CreateOp = {
21
+ data: Record<string, unknown>;
22
+ sourceIndex: number;
23
+ };
24
+ export type UpdateOp = {
25
+ id: string;
26
+ data: Record<string, unknown>;
27
+ before: Record<string, unknown>;
28
+ sourceIndex: number;
29
+ };
30
+ /** Exactly what to execute. Nothing here needs interpreting. */
31
+ export type WriteOperations = {
32
+ creates: CreateOp[];
33
+ updates: UpdateOp[];
34
+ /** Rows to remove — from `replace`, or from the `delete` missing policy. */
35
+ deletes: ExistingRow[];
36
+ /** Rows to mark inactive because the source no longer carries them. */
37
+ deactivates: ExistingRow[];
38
+ issues: RunIssue[];
39
+ warnings: string[];
40
+ };
41
+ export type PlanInput = {
42
+ records: readonly MappedRecord[];
43
+ /** Every row currently in the target table. */
44
+ existing: readonly ExistingRow[];
45
+ /** Target column that identifies the same record across runs, if configured. */
46
+ dedupTargetField: string | null;
47
+ writeMode: WriteMode;
48
+ missingPolicy: MissingRecordPolicy;
49
+ /** Column and value written when the policy is `deactivate`. */
50
+ missingColumn?: string | null;
51
+ missingValue?: string | null;
52
+ /** NOT NULL, no-default columns of the target — a row missing one cannot be written. */
53
+ requiredFields?: readonly string[];
54
+ };
55
+ /**
56
+ * Works out every write a run would perform.
57
+ *
58
+ * Records are processed in source order so an issue's `sourceIndex` always points
59
+ * at the record an operator can find in the sample.
60
+ */
61
+ export declare function planWrites(input: PlanInput): WriteOperations;
62
+ /** Applies the deactivation column/value to a row's update payload. */
63
+ export declare function deactivationPayload(missingColumn: string, missingValue: string): Record<string, unknown>;
64
+ /** Turns the operations into the summary a preview screen renders. */
65
+ export declare function summarize(ops: WriteOperations, meta: {
66
+ targetEntity: string;
67
+ writeMode: WriteMode;
68
+ missingPolicy: MissingRecordPolicy;
69
+ totalFetched: number;
70
+ reportedTotal: number | null;
71
+ existingCount: number;
72
+ dedupConfigured: boolean;
73
+ sampleSize?: number;
74
+ }): RunPlan;
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Undoing a run that already succeeded.
3
+ *
4
+ * A failed run needs nothing undone — its transaction rolled back. This is for
5
+ * the other case: the import worked, and afterwards somebody noticed the data was
6
+ * wrong. Bad records at the source, a mapping pointed at the wrong column, the
7
+ * wrong endpoint. Until now there was no way back from that at all; a wipe-and-
8
+ * replace ran and the previous contents were simply gone.
9
+ *
10
+ * ## Reversing, not re-importing
11
+ *
12
+ * Each before-image is turned back into its own opposite: a row this run created
13
+ * is deleted, a row it changed is written back to what it was, a row it deleted
14
+ * is put back. All in one transaction, so a revert is as all-or-nothing as the
15
+ * run it undoes.
16
+ *
17
+ * ## Somebody else's edit is not ours to discard
18
+ *
19
+ * Between the run and the revert, a person may have edited one of these rows by
20
+ * hand. Writing the before-image over that would destroy their work while
21
+ * appearing to be a safe undo. So a row whose current state no longer matches
22
+ * what the run left behind is reported as a conflict and skipped — unless the
23
+ * operator explicitly asks to overwrite it, having been told which rows.
24
+ */
25
+ import type { PreImage, WriterDeps } from './writer';
26
+ export type RevertOperations = {
27
+ /** Rows the run created, to be removed again. */
28
+ deletes: string[];
29
+ /** Rows to be written back to their previous contents. */
30
+ restores: {
31
+ id: string;
32
+ row: Record<string, unknown>;
33
+ }[];
34
+ /** Rows the run deleted, to be inserted again. */
35
+ reinserts: Record<string, unknown>[];
36
+ /** Rows changed since the run; not touched. */
37
+ conflicts: {
38
+ id: string;
39
+ reason: string;
40
+ }[];
41
+ };
42
+ export type RevertPlanInput = {
43
+ preImages: readonly PreImage[];
44
+ /** Current contents of every row the run touched, by id. Absent = already gone. */
45
+ current: ReadonlyMap<string, Record<string, unknown>>;
46
+ /** Overwrite rows that changed since the run, having reported them first. */
47
+ force?: boolean;
48
+ /** Columns excluded from the changed-since check — they move on every write. */
49
+ ignoreColumns?: readonly string[];
50
+ };
51
+ /**
52
+ * Works out how to undo a run. Pure — every rule about whose data may be
53
+ * overwritten is decided here and testable without a database.
54
+ */
55
+ export declare function planRevert(input: RevertPlanInput): RevertOperations;
56
+ export type RevertResult = {
57
+ ok: true;
58
+ deleted: number;
59
+ restored: number;
60
+ reinserted: number;
61
+ conflicts: number;
62
+ } | {
63
+ ok: false;
64
+ error: string;
65
+ conflicts: number;
66
+ };
67
+ /**
68
+ * Applies a revert plan in one transaction.
69
+ *
70
+ * Order mirrors the run it undoes: removals first, so an identifier or unique
71
+ * value freed here is available to the re-inserts that follow.
72
+ */
73
+ export declare function applyRevert(deps: WriterDeps, ops: RevertOperations, meta: {
74
+ targetEntity: string;
75
+ userId: string;
76
+ }): Promise<RevertResult>;
77
+ /** Ids a caller must read the current state of before planning a revert. */
78
+ export declare function touchedRowIds(preImages: readonly PreImage[]): string[];
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Running an import, start to finish.
3
+ *
4
+ * The order below is the whole design. Everything is fetched, mapped, planned and
5
+ * validated before a single row is written, and the write itself is one
6
+ * transaction. There is no state in which a run has half-happened.
7
+ *
8
+ * ```
9
+ * lock one run per mapping, across replicas
10
+ * collect every endpoint, in dependency order, all pages
11
+ * completeness refuse a destructive run that did not read everything
12
+ * map external records → our columns
13
+ * plan creates / updates / deletes / deactivations
14
+ * apply ONE transaction; any failure unwinds it
15
+ * record before-images, counts, and the run row
16
+ * ```
17
+ *
18
+ * Every step reports progress and checks for cancellation, because a chained
19
+ * import can legitimately take minutes and an operator who changes their mind
20
+ * must be able to stop it — not merely close the tab and hope.
21
+ */
22
+ import type { Logger } from '../Logger';
23
+ import { type EndpointSpec } from './collect';
24
+ import type { FetchImpl, SourceConnection } from './fetcher';
25
+ import { type OAuthConfig, type TokenCache } from './oauth';
26
+ import { type ExistingRow } from './plan';
27
+ import type { FieldMapping, MissingRecordPolicy, RunFailure, RunIssue, RunPlan, RunProgress, WriteMode } from './types';
28
+ import { type PreImage, type WriterDeps } from './writer';
29
+ /** The mapping being run, resolved from the database by the caller. */
30
+ export type MappingSpec = {
31
+ id: string;
32
+ name: string;
33
+ targetEntity: string;
34
+ fieldMappings: FieldMapping[];
35
+ dedupTargetField: string | null;
36
+ writeMode: WriteMode;
37
+ missingPolicy: MissingRecordPolicy;
38
+ missingColumn?: string | null;
39
+ missingValue?: string | null;
40
+ };
41
+ /** Everything a run needs, already read and decrypted. */
42
+ export type RunContext = {
43
+ mapping: MappingSpec;
44
+ /** The endpoint this mapping reads, plus every endpoint it depends on. */
45
+ endpoints: EndpointSpec[];
46
+ /** Which of `endpoints` the mapping itself maps from. */
47
+ rootEndpointId: string;
48
+ source: SourceConnection;
49
+ /** Present only when the source authenticates with OAuth2. */
50
+ oauth?: OAuthConfig | null;
51
+ userId: string;
52
+ trigger: 'manual' | 'schedule' | 'api';
53
+ };
54
+ /** Persistence and lookups the runner needs. Injected so it can be tested. */
55
+ export type RunRepository = {
56
+ openRun(input: {
57
+ mappingId: string;
58
+ targetEntity: string;
59
+ writeMode: WriteMode;
60
+ trigger: string;
61
+ userId: string;
62
+ }): Promise<{
63
+ ok: true;
64
+ runId: string;
65
+ } | {
66
+ ok: false;
67
+ error: string;
68
+ }>;
69
+ closeRun(runId: string, patch: Record<string, unknown>): Promise<{
70
+ ok: true;
71
+ } | {
72
+ ok: false;
73
+ error: string;
74
+ }>;
75
+ savePreImages(runId: string, images: PreImage[]): Promise<void>;
76
+ /** True once a cancel request has been recorded for this run. */
77
+ isCancelRequested(runId: string): Promise<boolean>;
78
+ /** Every row currently in the target table. */
79
+ readExistingRows(targetEntity: string): Promise<ExistingRow[]>;
80
+ /** Distinct values of one column, for an `entity` parameter binding. */
81
+ readEntityColumn(entity: string, field: string): Promise<unknown[]>;
82
+ /** NOT NULL, no-default columns of the target. */
83
+ requiredFields(targetEntity: string): string[];
84
+ };
85
+ export type RunnerDeps = {
86
+ repo: RunRepository;
87
+ writer: WriterDeps;
88
+ logger: Logger;
89
+ tokenCache: TokenCache;
90
+ fetchImpl?: FetchImpl;
91
+ publishProgress?: (progress: RunProgress) => void;
92
+ /**
93
+ * Called once the run record exists, before any work starts.
94
+ *
95
+ * A caller that answers 202 needs the identifier immediately and must not wait
96
+ * for the import. Racing a timer against the run would be a guess; this is the
97
+ * moment the identifier actually becomes real.
98
+ */
99
+ onRunOpened?: (runId: string) => void;
100
+ maxRecordsPerRun?: number;
101
+ maxConcurrentFanout?: number;
102
+ retainPreImagesMs?: number;
103
+ /** External cancellation, e.g. the process shutting down. */
104
+ abortSignal?: AbortSignal;
105
+ };
106
+ export type RunOutcome = {
107
+ runId: string | null;
108
+ status: 'succeeded' | 'failed' | 'cancelled';
109
+ failure: RunFailure | null;
110
+ plan: RunPlan | null;
111
+ created: number;
112
+ updated: number;
113
+ deleted: number;
114
+ deactivated: number;
115
+ externalCalls: number;
116
+ issues: RunIssue[];
117
+ };
118
+ /**
119
+ * Fetches everything, decides everything, then writes it once.
120
+ *
121
+ * Returns rather than throws: a failed import is an outcome an operator reads,
122
+ * not an exception for a route handler to translate.
123
+ */
124
+ export declare function executeRun(context: RunContext, deps: RunnerDeps, lock: {
125
+ isHeld: () => boolean;
126
+ }): Promise<RunOutcome>;
@@ -0,0 +1 @@
1
+ export {};