@x12i/memorix-service 3.3.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -6
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +10 -0
- package/dist/app.js.map +1 -1
- package/dist/blob-store/config.d.ts +31 -0
- package/dist/blob-store/config.d.ts.map +1 -0
- package/dist/blob-store/config.js +68 -0
- package/dist/blob-store/config.js.map +1 -0
- package/dist/blob-store/disk.d.ts +10 -0
- package/dist/blob-store/disk.d.ts.map +1 -0
- package/dist/blob-store/disk.js +104 -0
- package/dist/blob-store/disk.js.map +1 -0
- package/dist/blob-store/index.d.ts +6 -0
- package/dist/blob-store/index.d.ts.map +1 -0
- package/dist/blob-store/index.js +6 -0
- package/dist/blob-store/index.js.map +1 -0
- package/dist/blob-store/memory.d.ts +4 -0
- package/dist/blob-store/memory.d.ts.map +1 -0
- package/dist/blob-store/memory.js +47 -0
- package/dist/blob-store/memory.js.map +1 -0
- package/dist/blob-store/s3.d.ts +52 -0
- package/dist/blob-store/s3.d.ts.map +1 -0
- package/dist/blob-store/s3.js +151 -0
- package/dist/blob-store/s3.js.map +1 -0
- package/dist/blob-store/types.d.ts +30 -0
- package/dist/blob-store/types.d.ts.map +1 -0
- package/dist/blob-store/types.js +38 -0
- package/dist/blob-store/types.js.map +1 -0
- package/dist/catalog-store-cli.d.ts.map +1 -1
- package/dist/catalog-store-cli.js +35 -4
- package/dist/catalog-store-cli.js.map +1 -1
- package/dist/connector-ops/checkpoint-store.d.ts +7 -0
- package/dist/connector-ops/checkpoint-store.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-store.js +135 -0
- package/dist/connector-ops/checkpoint-store.js.map +1 -0
- package/dist/connector-ops/checkpoint-types.d.ts +46 -0
- package/dist/connector-ops/checkpoint-types.d.ts.map +1 -0
- package/dist/connector-ops/checkpoint-types.js +14 -0
- package/dist/connector-ops/checkpoint-types.js.map +1 -0
- package/dist/connector-ops/continuous-types.d.ts +141 -0
- package/dist/connector-ops/continuous-types.d.ts.map +1 -0
- package/dist/connector-ops/continuous-types.js +13 -0
- package/dist/connector-ops/continuous-types.js.map +1 -0
- package/dist/connector-ops/coordinator.d.ts +88 -0
- package/dist/connector-ops/coordinator.d.ts.map +1 -0
- package/dist/connector-ops/coordinator.js +174 -0
- package/dist/connector-ops/coordinator.js.map +1 -0
- package/dist/connector-ops/index.d.ts +6 -0
- package/dist/connector-ops/index.d.ts.map +1 -1
- package/dist/connector-ops/index.js +6 -0
- package/dist/connector-ops/index.js.map +1 -1
- package/dist/connector-ops/run-store.d.ts +16 -0
- package/dist/connector-ops/run-store.d.ts.map +1 -0
- package/dist/connector-ops/run-store.js +249 -0
- package/dist/connector-ops/run-store.js.map +1 -0
- package/dist/connector-ops/schedule-lease-store.d.ts +8 -0
- package/dist/connector-ops/schedule-lease-store.d.ts.map +1 -0
- package/dist/connector-ops/schedule-lease-store.js +318 -0
- package/dist/connector-ops/schedule-lease-store.js.map +1 -0
- package/dist/incremental-state/index.d.ts +3 -0
- package/dist/incremental-state/index.d.ts.map +1 -0
- package/dist/incremental-state/index.js +3 -0
- package/dist/incremental-state/index.js.map +1 -0
- package/dist/incremental-state/state-store.d.ts +7 -0
- package/dist/incremental-state/state-store.d.ts.map +1 -0
- package/dist/incremental-state/state-store.js +183 -0
- package/dist/incremental-state/state-store.js.map +1 -0
- package/dist/incremental-state/types.d.ts +104 -0
- package/dist/incremental-state/types.d.ts.map +1 -0
- package/dist/incremental-state/types.js +8 -0
- package/dist/incremental-state/types.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/integrations/index.d.ts +3 -0
- package/dist/integrations/index.d.ts.map +1 -0
- package/dist/integrations/index.js +3 -0
- package/dist/integrations/index.js.map +1 -0
- package/dist/integrations/instance-store.d.ts +5 -0
- package/dist/integrations/instance-store.d.ts.map +1 -0
- package/dist/integrations/instance-store.js +327 -0
- package/dist/integrations/instance-store.js.map +1 -0
- package/dist/integrations/types.d.ts +92 -0
- package/dist/integrations/types.d.ts.map +1 -0
- package/dist/integrations/types.js +40 -0
- package/dist/integrations/types.js.map +1 -0
- package/dist/magit/magit-pack-service.d.ts +187 -3
- package/dist/magit/magit-pack-service.d.ts.map +1 -1
- package/dist/magit/magit-pack-service.js +401 -8
- package/dist/magit/magit-pack-service.js.map +1 -1
- package/dist/openapi/openapi.json +1348 -2
- package/dist/platform.d.ts +44 -1
- package/dist/platform.d.ts.map +1 -1
- package/dist/platform.js +172 -1
- package/dist/platform.js.map +1 -1
- package/dist/restricted-store/crypto.d.ts +10 -0
- package/dist/restricted-store/crypto.d.ts.map +1 -0
- package/dist/restricted-store/crypto.js +43 -0
- package/dist/restricted-store/crypto.js.map +1 -0
- package/dist/restricted-store/index.d.ts +4 -0
- package/dist/restricted-store/index.d.ts.map +1 -0
- package/dist/restricted-store/index.js +4 -0
- package/dist/restricted-store/index.js.map +1 -0
- package/dist/restricted-store/object-store.d.ts +12 -0
- package/dist/restricted-store/object-store.d.ts.map +1 -0
- package/dist/restricted-store/object-store.js +233 -0
- package/dist/restricted-store/object-store.js.map +1 -0
- package/dist/restricted-store/types.d.ts +78 -0
- package/dist/restricted-store/types.d.ts.map +1 -0
- package/dist/restricted-store/types.js +12 -0
- package/dist/restricted-store/types.js.map +1 -0
- package/dist/routes/continuous-acquisition.d.ts +5 -0
- package/dist/routes/continuous-acquisition.d.ts.map +1 -0
- package/dist/routes/continuous-acquisition.js +259 -0
- package/dist/routes/continuous-acquisition.js.map +1 -0
- package/dist/routes/incremental-state.d.ts +5 -0
- package/dist/routes/incremental-state.d.ts.map +1 -0
- package/dist/routes/incremental-state.js +173 -0
- package/dist/routes/incremental-state.js.map +1 -0
- package/dist/routes/integrations.d.ts +5 -0
- package/dist/routes/integrations.d.ts.map +1 -0
- package/dist/routes/integrations.js +126 -0
- package/dist/routes/integrations.js.map +1 -0
- package/dist/routes/metadata.d.ts.map +1 -1
- package/dist/routes/metadata.js +328 -4
- package/dist/routes/metadata.js.map +1 -1
- package/dist/routes/restricted-store.d.ts +5 -0
- package/dist/routes/restricted-store.d.ts.map +1 -0
- package/dist/routes/restricted-store.js +117 -0
- package/dist/routes/restricted-store.js.map +1 -0
- package/openapi/openapi.json +1348 -2
- package/package.json +3 -2
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
import type { StreamCheckpoint } from "./checkpoint-types.js";
|
|
6
|
+
export type AcquisitionCadence = {
|
|
7
|
+
kind: "interval";
|
|
8
|
+
intervalSeconds: number;
|
|
9
|
+
} | {
|
|
10
|
+
kind: "cron";
|
|
11
|
+
expression: string;
|
|
12
|
+
};
|
|
13
|
+
export type StreamSchedule = {
|
|
14
|
+
orgId: string;
|
|
15
|
+
connectorId: string;
|
|
16
|
+
instanceId: string;
|
|
17
|
+
streamId: string;
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
cadence: AcquisitionCadence;
|
|
20
|
+
/** Soft bounds passed to bounded /pipeline invokes. */
|
|
21
|
+
bounds?: {
|
|
22
|
+
maxDurationMs?: number;
|
|
23
|
+
maxPages?: number;
|
|
24
|
+
maxEvents?: number;
|
|
25
|
+
};
|
|
26
|
+
updatedAt: string;
|
|
27
|
+
updatedBy?: string;
|
|
28
|
+
};
|
|
29
|
+
export type PutStreamSchedule = {
|
|
30
|
+
enabled?: boolean;
|
|
31
|
+
cadence: AcquisitionCadence;
|
|
32
|
+
bounds?: StreamSchedule["bounds"];
|
|
33
|
+
updatedBy?: string;
|
|
34
|
+
};
|
|
35
|
+
export type StreamLease = {
|
|
36
|
+
orgId: string;
|
|
37
|
+
connectorId: string;
|
|
38
|
+
instanceId: string;
|
|
39
|
+
streamId: string;
|
|
40
|
+
ownerId: string;
|
|
41
|
+
leaseId: string;
|
|
42
|
+
acquiredAt: string;
|
|
43
|
+
expiresAt: string;
|
|
44
|
+
renewCount: number;
|
|
45
|
+
};
|
|
46
|
+
export type AcquireLeaseInput = {
|
|
47
|
+
ownerId: string;
|
|
48
|
+
/** Lease TTL in ms (default 60_000). */
|
|
49
|
+
ttlMs?: number;
|
|
50
|
+
};
|
|
51
|
+
export type AcquisitionRunOutcome = "ok" | "fail" | "skip" | "canceled" | "lease_lost";
|
|
52
|
+
export type AcquisitionRunRecord = {
|
|
53
|
+
orgId: string;
|
|
54
|
+
connectorId: string;
|
|
55
|
+
instanceId: string;
|
|
56
|
+
streamId: string;
|
|
57
|
+
runId: string;
|
|
58
|
+
startedAt: string;
|
|
59
|
+
endedAt?: string | null;
|
|
60
|
+
outcome?: AcquisitionRunOutcome | null;
|
|
61
|
+
errorClass?: string | null;
|
|
62
|
+
errorMessage?: string | null;
|
|
63
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
64
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
65
|
+
bounds?: StreamSchedule["bounds"] | null;
|
|
66
|
+
ownerId?: string | null;
|
|
67
|
+
cancelRequested?: boolean;
|
|
68
|
+
};
|
|
69
|
+
export type StartAcquisitionRunInput = {
|
|
70
|
+
runId?: string;
|
|
71
|
+
ownerId?: string;
|
|
72
|
+
bounds?: StreamSchedule["bounds"];
|
|
73
|
+
checkpointBefore?: Partial<StreamCheckpoint> | null;
|
|
74
|
+
};
|
|
75
|
+
export type EndAcquisitionRunInput = {
|
|
76
|
+
outcome: AcquisitionRunOutcome;
|
|
77
|
+
errorClass?: string;
|
|
78
|
+
errorMessage?: string;
|
|
79
|
+
checkpointAfter?: Partial<StreamCheckpoint> | null;
|
|
80
|
+
};
|
|
81
|
+
export type StreamFreshness = {
|
|
82
|
+
orgId: string;
|
|
83
|
+
connectorId: string;
|
|
84
|
+
instanceId: string;
|
|
85
|
+
streamId: string;
|
|
86
|
+
scheduleEnabled: boolean;
|
|
87
|
+
lastSuccessAt?: string | null;
|
|
88
|
+
lastFailAt?: string | null;
|
|
89
|
+
lastRunAt?: string | null;
|
|
90
|
+
lastOutcome?: AcquisitionRunOutcome | null;
|
|
91
|
+
/** Seconds since last success; null if never succeeded. */
|
|
92
|
+
lagSeconds?: number | null;
|
|
93
|
+
/** True when lag exceeds 2× cadence interval (or 1h for cron). */
|
|
94
|
+
aging?: boolean;
|
|
95
|
+
/** True when no success within 3× cadence (or 3h for cron). */
|
|
96
|
+
lapsed?: boolean;
|
|
97
|
+
activeLeaseOwnerId?: string | null;
|
|
98
|
+
activeLeaseExpiresAt?: string | null;
|
|
99
|
+
cancelRequested?: boolean;
|
|
100
|
+
};
|
|
101
|
+
export type StreamScheduleStore = {
|
|
102
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamSchedule | null>;
|
|
103
|
+
put(connectorId: string, instanceId: string, streamId: string, body: PutStreamSchedule): Promise<StreamSchedule>;
|
|
104
|
+
clear(connectorId: string, instanceId: string, streamId: string): Promise<boolean>;
|
|
105
|
+
listEnabled(opts?: {
|
|
106
|
+
connectorId?: string;
|
|
107
|
+
instanceId?: string;
|
|
108
|
+
}): Promise<StreamSchedule[]>;
|
|
109
|
+
};
|
|
110
|
+
export type StreamLeaseStore = {
|
|
111
|
+
get(connectorId: string, instanceId: string, streamId: string): Promise<StreamLease | null>;
|
|
112
|
+
acquire(connectorId: string, instanceId: string, streamId: string, input: AcquireLeaseInput): Promise<StreamLease>;
|
|
113
|
+
renew(connectorId: string, instanceId: string, streamId: string, input: {
|
|
114
|
+
ownerId: string;
|
|
115
|
+
leaseId: string;
|
|
116
|
+
ttlMs?: number;
|
|
117
|
+
}): Promise<StreamLease>;
|
|
118
|
+
release(connectorId: string, instanceId: string, streamId: string, input: {
|
|
119
|
+
ownerId: string;
|
|
120
|
+
leaseId: string;
|
|
121
|
+
}): Promise<boolean>;
|
|
122
|
+
};
|
|
123
|
+
export type AcquisitionRunStore = {
|
|
124
|
+
start(connectorId: string, instanceId: string, streamId: string, input: StartAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
125
|
+
end(connectorId: string, instanceId: string, streamId: string, runId: string, input: EndAcquisitionRunInput): Promise<AcquisitionRunRecord>;
|
|
126
|
+
get(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
127
|
+
requestCancel(connectorId: string, instanceId: string, streamId: string, runId: string): Promise<AcquisitionRunRecord | null>;
|
|
128
|
+
list(connectorId: string, instanceId: string, streamId: string, opts?: {
|
|
129
|
+
limit?: number;
|
|
130
|
+
}): Promise<AcquisitionRunRecord[]>;
|
|
131
|
+
latestSuccess(connectorId: string, instanceId: string, streamId: string): Promise<AcquisitionRunRecord | null>;
|
|
132
|
+
};
|
|
133
|
+
export declare const CONT_COLLECTIONS: {
|
|
134
|
+
readonly schedules: "mco_stream_schedules";
|
|
135
|
+
readonly leases: "mco_stream_leases";
|
|
136
|
+
readonly runs: "mco_acquisition_runs";
|
|
137
|
+
};
|
|
138
|
+
export declare const DEFAULT_LEASE_TTL_MS = 60000;
|
|
139
|
+
export declare const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
140
|
+
export declare const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
141
|
+
//# sourceMappingURL=continuous-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.d.ts","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzC,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,uDAAuD;IACvD,MAAM,CAAC,EAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAC7B,IAAI,GACJ,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,CAAC;AAEjB,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACpD,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;IACnD,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACrD,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,qBAAqB,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;CACpD,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC3C,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,kEAAkE;IAClE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAClC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,iBAAiB,GACtB,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,WAAW,CAAC,IAAI,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;IAC/B,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,OAAO,CAAC,WAAW,CAAC,CAAC;IACxB,OAAO,CACL,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAC1C,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,CACH,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,sBAAsB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,GAAG,CACD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IACxC,IAAI,CACF,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACxB,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACnC,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;CACzC,CAAC;AAEF,eAAO,MAAM,gBAAgB;;;;CAInB,CAAC;AAEX,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAC3C,eAAO,MAAM,8BAA8B,KAAK,CAAC;AACjD,eAAO,MAAM,0BAA0B,MAAM,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MX-CONT-01 continuous acquisition — schedule, lease, run ledger, freshness.
|
|
3
|
+
* Product-plane under connector-ops; not MGC.
|
|
4
|
+
*/
|
|
5
|
+
export const CONT_COLLECTIONS = {
|
|
6
|
+
schedules: "mco_stream_schedules",
|
|
7
|
+
leases: "mco_stream_leases",
|
|
8
|
+
runs: "mco_acquisition_runs",
|
|
9
|
+
};
|
|
10
|
+
export const DEFAULT_LEASE_TTL_MS = 60_000;
|
|
11
|
+
export const ACQUISITION_RUNS_DEFAULT_LIMIT = 50;
|
|
12
|
+
export const ACQUISITION_RUNS_MAX_LIMIT = 200;
|
|
13
|
+
//# sourceMappingURL=continuous-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"continuous-types.js","sourceRoot":"","sources":["../../src/connector-ops/continuous-types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsMH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,SAAS,EAAE,sBAAsB;IACjC,MAAM,EAAE,mBAAmB;IAC3B,IAAI,EAAE,sBAAsB;CACpB,CAAC;AAEX,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC3C,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AACjD,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs-ticked continuous acquisition coordinator (MX-CONT-01 host).
|
|
3
|
+
* Bounded pipeline invokes only — no always-on CF-A5 fleet daemon.
|
|
4
|
+
*/
|
|
5
|
+
import type { IntegrationInstanceStore } from "../integrations/types.js";
|
|
6
|
+
import type { McoOverrideStore } from "./types.js";
|
|
7
|
+
import type { StreamCheckpointStore } from "./checkpoint-types.js";
|
|
8
|
+
import type { AcquisitionRunStore, StreamLeaseStore, StreamSchedule, StreamScheduleStore } from "./continuous-types.js";
|
|
9
|
+
export type PipelineInvokeBounds = {
|
|
10
|
+
maxDurationMs?: number;
|
|
11
|
+
maxPages?: number;
|
|
12
|
+
maxEvents?: number;
|
|
13
|
+
};
|
|
14
|
+
export type BoundedPipelineInvokeResult = {
|
|
15
|
+
ok: boolean;
|
|
16
|
+
applyAck: boolean;
|
|
17
|
+
errorClass?: string;
|
|
18
|
+
errorMessage?: string;
|
|
19
|
+
/** Checkpoint payload to commit only when applyAck is true. */
|
|
20
|
+
nextCheckpoint?: {
|
|
21
|
+
method: "watermark" | "revision" | "fullScanHash" | "cursor";
|
|
22
|
+
watermark?: string | number | null;
|
|
23
|
+
revision?: string | number | null;
|
|
24
|
+
fullScanHash?: string | null;
|
|
25
|
+
cursor?: string | null;
|
|
26
|
+
contentHash?: string | null;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export type BoundedPipelineInvoker = (input: {
|
|
30
|
+
orgId: string;
|
|
31
|
+
connectorId: string;
|
|
32
|
+
instanceId: string;
|
|
33
|
+
streamId: string;
|
|
34
|
+
schedule: StreamSchedule;
|
|
35
|
+
bounds: PipelineInvokeBounds;
|
|
36
|
+
runId: string;
|
|
37
|
+
checkpoint: Awaited<ReturnType<StreamCheckpointStore["get"]>>;
|
|
38
|
+
abortSignal?: AbortSignal;
|
|
39
|
+
}) => Promise<BoundedPipelineInvokeResult>;
|
|
40
|
+
export type ContinuousAcquisitionTickDeps = {
|
|
41
|
+
orgId: string;
|
|
42
|
+
ownerId: string;
|
|
43
|
+
schedules: StreamScheduleStore;
|
|
44
|
+
leases: StreamLeaseStore;
|
|
45
|
+
runs: AcquisitionRunStore;
|
|
46
|
+
checkpoints: StreamCheckpointStore;
|
|
47
|
+
integrations: IntegrationInstanceStore;
|
|
48
|
+
overrides: McoOverrideStore;
|
|
49
|
+
invokePipeline: BoundedPipelineInvoker;
|
|
50
|
+
/** Optional op id used for MCO kill-switch (default continuous-acquire). */
|
|
51
|
+
mcoOpId?: string;
|
|
52
|
+
leaseTtlMs?: number;
|
|
53
|
+
};
|
|
54
|
+
export type ContinuousAcquisitionTickResult = {
|
|
55
|
+
orgId: string;
|
|
56
|
+
considered: number;
|
|
57
|
+
started: number;
|
|
58
|
+
skipped: number;
|
|
59
|
+
succeeded: number;
|
|
60
|
+
failed: number;
|
|
61
|
+
details: Array<{
|
|
62
|
+
connectorId: string;
|
|
63
|
+
instanceId: string;
|
|
64
|
+
streamId: string;
|
|
65
|
+
outcome: string;
|
|
66
|
+
reason?: string;
|
|
67
|
+
runId?: string;
|
|
68
|
+
}>;
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Profile helpers for acceptance proofs (CF-A5 vs Splunk watermark window).
|
|
72
|
+
*/
|
|
73
|
+
export declare const COORDINATOR_PROFILES: {
|
|
74
|
+
/** CrowdStrike CF-A5-style: hard event/partition cap. */
|
|
75
|
+
readonly cfA5: {
|
|
76
|
+
maxEvents: number;
|
|
77
|
+
maxPages: number;
|
|
78
|
+
maxDurationMs: number;
|
|
79
|
+
};
|
|
80
|
+
/** Splunk-style watermark window (bounded lookback pages). */
|
|
81
|
+
readonly splunkWatermark: {
|
|
82
|
+
maxPages: number;
|
|
83
|
+
maxEvents: number;
|
|
84
|
+
maxDurationMs: number;
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
export declare function tickContinuousAcquisition(deps: ContinuousAcquisitionTickDeps): Promise<ContinuousAcquisitionTickResult>;
|
|
88
|
+
//# sourceMappingURL=coordinator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/connector-ops/coordinator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,oBAAoB,GAAG;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,cAAc,CAAC,EAAE;QACf,MAAM,EAAE,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,QAAQ,CAAC;QAC7D,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QACnC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;QAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CAAC,KAAK,EAAE;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9D,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAE3C,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,mBAAmB,CAAC;IAC/B,MAAM,EAAE,gBAAgB,CAAC;IACzB,IAAI,EAAE,mBAAmB,CAAC;IAC1B,WAAW,EAAE,qBAAqB,CAAC;IACnC,YAAY,EAAE,wBAAwB,CAAC;IACvC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,cAAc,EAAE,sBAAsB,CAAC;IACvC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;IAC/B,yDAAyD;;;;;;IAMzD,8DAA8D;;;;;;CAMtD,CAAC;AAEX,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,6BAA6B,GAClC,OAAO,CAAC,+BAA+B,CAAC,CA4K1C"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs-ticked continuous acquisition coordinator (MX-CONT-01 host).
|
|
3
|
+
* Bounded pipeline invokes only — no always-on CF-A5 fleet daemon.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Profile helpers for acceptance proofs (CF-A5 vs Splunk watermark window).
|
|
7
|
+
*/
|
|
8
|
+
export const COORDINATOR_PROFILES = {
|
|
9
|
+
/** CrowdStrike CF-A5-style: hard event/partition cap. */
|
|
10
|
+
cfA5: {
|
|
11
|
+
maxEvents: 500,
|
|
12
|
+
maxPages: 10,
|
|
13
|
+
maxDurationMs: 120_000,
|
|
14
|
+
},
|
|
15
|
+
/** Splunk-style watermark window (bounded lookback pages). */
|
|
16
|
+
splunkWatermark: {
|
|
17
|
+
maxPages: 5,
|
|
18
|
+
maxEvents: 5_000,
|
|
19
|
+
maxDurationMs: 180_000,
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
export async function tickContinuousAcquisition(deps) {
|
|
23
|
+
const mcoOpId = deps.mcoOpId ?? "continuous-acquire";
|
|
24
|
+
const enabled = await deps.schedules.listEnabled();
|
|
25
|
+
const result = {
|
|
26
|
+
orgId: deps.orgId,
|
|
27
|
+
considered: enabled.length,
|
|
28
|
+
started: 0,
|
|
29
|
+
skipped: 0,
|
|
30
|
+
succeeded: 0,
|
|
31
|
+
failed: 0,
|
|
32
|
+
details: [],
|
|
33
|
+
};
|
|
34
|
+
for (const schedule of enabled) {
|
|
35
|
+
const { connectorId, instanceId, streamId } = schedule;
|
|
36
|
+
const detailBase = { connectorId, instanceId, streamId };
|
|
37
|
+
const instance = await deps.integrations.get(instanceId);
|
|
38
|
+
if (!instance || instance.enabled === false) {
|
|
39
|
+
result.skipped += 1;
|
|
40
|
+
result.details.push({
|
|
41
|
+
...detailBase,
|
|
42
|
+
outcome: "skip",
|
|
43
|
+
reason: "instance_disabled",
|
|
44
|
+
});
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const override = await deps.overrides.getOverride(connectorId, mcoOpId);
|
|
48
|
+
if (override && override.enabled === false) {
|
|
49
|
+
result.skipped += 1;
|
|
50
|
+
result.details.push({
|
|
51
|
+
...detailBase,
|
|
52
|
+
outcome: "skip",
|
|
53
|
+
reason: "mco_disabled",
|
|
54
|
+
});
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
let lease;
|
|
58
|
+
try {
|
|
59
|
+
lease = await deps.leases.acquire(connectorId, instanceId, streamId, {
|
|
60
|
+
ownerId: deps.ownerId,
|
|
61
|
+
ttlMs: deps.leaseTtlMs,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
result.skipped += 1;
|
|
66
|
+
result.details.push({
|
|
67
|
+
...detailBase,
|
|
68
|
+
outcome: "skip",
|
|
69
|
+
reason: err instanceof Error && "code" in err
|
|
70
|
+
? String(err.code)
|
|
71
|
+
: "lease_held",
|
|
72
|
+
});
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const checkpointBefore = await deps.checkpoints.get(connectorId, instanceId, streamId);
|
|
76
|
+
const bounds = {
|
|
77
|
+
...COORDINATOR_PROFILES.cfA5,
|
|
78
|
+
...schedule.bounds,
|
|
79
|
+
};
|
|
80
|
+
const run = await deps.runs.start(connectorId, instanceId, streamId, {
|
|
81
|
+
ownerId: deps.ownerId,
|
|
82
|
+
bounds,
|
|
83
|
+
checkpointBefore: checkpointBefore ?? null,
|
|
84
|
+
});
|
|
85
|
+
result.started += 1;
|
|
86
|
+
try {
|
|
87
|
+
const live = await deps.runs.get(connectorId, instanceId, streamId, run.runId);
|
|
88
|
+
if (live?.cancelRequested) {
|
|
89
|
+
await deps.runs.end(connectorId, instanceId, streamId, run.runId, {
|
|
90
|
+
outcome: "canceled",
|
|
91
|
+
errorClass: "canceled",
|
|
92
|
+
});
|
|
93
|
+
result.skipped += 1;
|
|
94
|
+
result.details.push({
|
|
95
|
+
...detailBase,
|
|
96
|
+
outcome: "canceled",
|
|
97
|
+
runId: run.runId,
|
|
98
|
+
});
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
const invoke = await deps.invokePipeline({
|
|
102
|
+
orgId: deps.orgId,
|
|
103
|
+
connectorId,
|
|
104
|
+
instanceId,
|
|
105
|
+
streamId,
|
|
106
|
+
schedule,
|
|
107
|
+
bounds,
|
|
108
|
+
runId: run.runId,
|
|
109
|
+
checkpoint: checkpointBefore,
|
|
110
|
+
});
|
|
111
|
+
if (!invoke.ok || !invoke.applyAck || !invoke.nextCheckpoint) {
|
|
112
|
+
await deps.runs.end(connectorId, instanceId, streamId, run.runId, {
|
|
113
|
+
outcome: "fail",
|
|
114
|
+
errorClass: invoke.errorClass ?? "apply_failed",
|
|
115
|
+
errorMessage: invoke.errorMessage,
|
|
116
|
+
checkpointAfter: checkpointBefore,
|
|
117
|
+
});
|
|
118
|
+
result.failed += 1;
|
|
119
|
+
result.details.push({
|
|
120
|
+
...detailBase,
|
|
121
|
+
outcome: "fail",
|
|
122
|
+
reason: invoke.errorClass ?? "apply_failed",
|
|
123
|
+
runId: run.runId,
|
|
124
|
+
});
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
// Checkpoint advances ONLY after durable apply ack.
|
|
128
|
+
const after = await deps.checkpoints.put(connectorId, instanceId, streamId, {
|
|
129
|
+
...invoke.nextCheckpoint,
|
|
130
|
+
producerRunId: run.runId,
|
|
131
|
+
expectedCheckpointRevision: checkpointBefore?.checkpointRevision,
|
|
132
|
+
});
|
|
133
|
+
await deps.runs.end(connectorId, instanceId, streamId, run.runId, {
|
|
134
|
+
outcome: "ok",
|
|
135
|
+
checkpointAfter: after,
|
|
136
|
+
});
|
|
137
|
+
result.succeeded += 1;
|
|
138
|
+
result.details.push({
|
|
139
|
+
...detailBase,
|
|
140
|
+
outcome: "ok",
|
|
141
|
+
runId: run.runId,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
146
|
+
await deps.runs.end(connectorId, instanceId, streamId, run.runId, {
|
|
147
|
+
outcome: "fail",
|
|
148
|
+
errorClass: "coordinator_error",
|
|
149
|
+
errorMessage: message,
|
|
150
|
+
checkpointAfter: checkpointBefore,
|
|
151
|
+
});
|
|
152
|
+
result.failed += 1;
|
|
153
|
+
result.details.push({
|
|
154
|
+
...detailBase,
|
|
155
|
+
outcome: "fail",
|
|
156
|
+
reason: "coordinator_error",
|
|
157
|
+
runId: run.runId,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
try {
|
|
162
|
+
await deps.leases.release(connectorId, instanceId, streamId, {
|
|
163
|
+
ownerId: lease.ownerId,
|
|
164
|
+
leaseId: lease.leaseId,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// lease may have expired; next tick can acquire
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
//# sourceMappingURL=coordinator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coordinator.js","sourceRoot":"","sources":["../../src/connector-ops/coordinator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8EH;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,yDAAyD;IACzD,IAAI,EAAE;QACJ,SAAS,EAAE,GAAG;QACd,QAAQ,EAAE,EAAE;QACZ,aAAa,EAAE,OAAO;KACQ;IAChC,8DAA8D;IAC9D,eAAe,EAAE;QACf,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,KAAK;QAChB,aAAa,EAAE,OAAO;KACQ;CACxB,CAAC;AAEX,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAmC;IAEnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,oBAAoB,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IACnD,MAAM,MAAM,GAAoC;QAC9C,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;QAC/B,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;QACvD,MAAM,UAAU,GAAG,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC5C,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB,GAAG,UAAU;gBACb,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,mBAAmB;aAC5B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACxE,IAAI,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC3C,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB,GAAG,UAAU;gBACb,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,cAAc;aACvB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE;gBACnE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,IAAI,CAAC,UAAU;aACvB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB,GAAG,UAAU;gBACb,OAAO,EAAE,MAAM;gBACf,MAAM,EACJ,GAAG,YAAY,KAAK,IAAI,MAAM,IAAI,GAAG;oBACnC,CAAC,CAAC,MAAM,CAAE,GAAwB,CAAC,IAAI,CAAC;oBACxC,CAAC,CAAC,YAAY;aACnB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CACjD,WAAW,EACX,UAAU,EACV,QAAQ,CACT,CAAC;QACF,MAAM,MAAM,GAAG;YACb,GAAG,oBAAoB,CAAC,IAAI;YAC5B,GAAG,QAAQ,CAAC,MAAM;SACnB,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE;YACnE,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM;YACN,gBAAgB,EAAE,gBAAgB,IAAI,IAAI;SAC3C,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QAEpB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAC9B,WAAW,EACX,UAAU,EACV,QAAQ,EACR,GAAG,CAAC,KAAK,CACV,CAAC;YACF,IAAI,IAAI,EAAE,eAAe,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE;oBAChE,OAAO,EAAE,UAAU;oBACnB,UAAU,EAAE,UAAU;iBACvB,CAAC,CAAC;gBACH,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;oBAClB,GAAG,UAAU;oBACb,OAAO,EAAE,UAAU;oBACnB,KAAK,EAAE,GAAG,CAAC,KAAK;iBACjB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACvC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW;gBACX,UAAU;gBACV,QAAQ;gBACR,QAAQ;gBACR,MAAM;gBACN,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,UAAU,EAAE,gBAAgB;aAC7B,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC7D,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE;oBAChE,OAAO,EAAE,MAAM;oBACf,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,cAAc;oBAC/C,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,eAAe,EAAE,gBAAgB;iBAClC,CAAC,CAAC;gBACH,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;oBAClB,GAAG,UAAU;oBACb,OAAO,EAAE,MAAM;oBACf,MAAM,EAAE,MAAM,CAAC,UAAU,IAAI,cAAc;oBAC3C,KAAK,EAAE,GAAG,CAAC,KAAK;iBACjB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,oDAAoD;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CACtC,WAAW,EACX,UAAU,EACV,QAAQ,EACR;gBACE,GAAG,MAAM,CAAC,cAAc;gBACxB,aAAa,EAAE,GAAG,CAAC,KAAK;gBACxB,0BAA0B,EAAE,gBAAgB,EAAE,kBAAkB;aACjE,CACF,CAAC;YACF,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE;gBAChE,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,KAAK;aACvB,CAAC,CAAC;YACH,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;YACtB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB,GAAG,UAAU;gBACb,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE;gBAChE,OAAO,EAAE,MAAM;gBACf,UAAU,EAAE,mBAAmB;gBAC/B,YAAY,EAAE,OAAO;gBACrB,eAAe,EAAE,gBAAgB;aAClC,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;YACnB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;gBAClB,GAAG,UAAU;gBACb,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE;oBAC3D,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,gDAAgD;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
export * from "./types.js";
|
|
2
2
|
export * from "./sanitize.js";
|
|
3
|
+
export * from "./checkpoint-types.js";
|
|
4
|
+
export * from "./continuous-types.js";
|
|
3
5
|
export { createInMemoryMcoOverrideStore, createMongoMcoOverrideStore, isOverrideActive, validateSetOverride, } from "./override-store.js";
|
|
4
6
|
export { createInMemoryMcoAttemptStore, createMongoMcoAttemptStore, } from "./attempt-store.js";
|
|
7
|
+
export { createInMemoryStreamCheckpointStore, createMongoStreamCheckpointStore, validatePutCheckpoint, assertCheckpointMethod, } from "./checkpoint-store.js";
|
|
8
|
+
export { createInMemoryStreamScheduleStore, createMongoStreamScheduleStore, createInMemoryStreamLeaseStore, createMongoStreamLeaseStore, validatePutSchedule, } from "./schedule-lease-store.js";
|
|
9
|
+
export { createInMemoryAcquisitionRunStore, createMongoAcquisitionRunStore, computeStreamFreshness, } from "./run-store.js";
|
|
10
|
+
export { tickContinuousAcquisition, COORDINATOR_PROFILES, type ContinuousAcquisitionTickDeps, type ContinuousAcquisitionTickResult, type BoundedPipelineInvoker, type BoundedPipelineInvokeResult, } from "./coordinator.js";
|
|
5
11
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/connector-ops/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,EAC3B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/connector-ops/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,EAC3B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mCAAmC,EACnC,gCAAgC,EAChC,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,iCAAiC,EACjC,8BAA8B,EAC9B,8BAA8B,EAC9B,2BAA2B,EAC3B,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,iCAAiC,EACjC,8BAA8B,EAC9B,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,yBAAyB,EACzB,oBAAoB,EACpB,KAAK,6BAA6B,EAClC,KAAK,+BAA+B,EACpC,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,GACjC,MAAM,kBAAkB,CAAC"}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
export * from "./types.js";
|
|
2
2
|
export * from "./sanitize.js";
|
|
3
|
+
export * from "./checkpoint-types.js";
|
|
4
|
+
export * from "./continuous-types.js";
|
|
3
5
|
export { createInMemoryMcoOverrideStore, createMongoMcoOverrideStore, isOverrideActive, validateSetOverride, } from "./override-store.js";
|
|
4
6
|
export { createInMemoryMcoAttemptStore, createMongoMcoAttemptStore, } from "./attempt-store.js";
|
|
7
|
+
export { createInMemoryStreamCheckpointStore, createMongoStreamCheckpointStore, validatePutCheckpoint, assertCheckpointMethod, } from "./checkpoint-store.js";
|
|
8
|
+
export { createInMemoryStreamScheduleStore, createMongoStreamScheduleStore, createInMemoryStreamLeaseStore, createMongoStreamLeaseStore, validatePutSchedule, } from "./schedule-lease-store.js";
|
|
9
|
+
export { createInMemoryAcquisitionRunStore, createMongoAcquisitionRunStore, computeStreamFreshness, } from "./run-store.js";
|
|
10
|
+
export { tickContinuousAcquisition, COORDINATOR_PROFILES, } from "./coordinator.js";
|
|
5
11
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/connector-ops/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,EAC3B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/connector-ops/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,EAC3B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mCAAmC,EACnC,gCAAgC,EAChC,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,iCAAiC,EACjC,8BAA8B,EAC9B,8BAA8B,EAC9B,2BAA2B,EAC3B,mBAAmB,GACpB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,iCAAiC,EACjC,8BAA8B,EAC9B,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,yBAAyB,EACzB,oBAAoB,GAKrB,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { MongoClient } from "mongodb";
|
|
2
|
+
import { type AcquisitionRunRecord, type AcquisitionRunStore, type StreamFreshness, type StreamLease, type StreamSchedule } from "./continuous-types.js";
|
|
3
|
+
export declare function createInMemoryAcquisitionRunStore(orgId: string): AcquisitionRunStore;
|
|
4
|
+
export declare function createMongoAcquisitionRunStore(client: MongoClient, orgId: string): AcquisitionRunStore;
|
|
5
|
+
export declare function computeStreamFreshness(input: {
|
|
6
|
+
orgId: string;
|
|
7
|
+
connectorId: string;
|
|
8
|
+
instanceId: string;
|
|
9
|
+
streamId: string;
|
|
10
|
+
schedule: StreamSchedule | null;
|
|
11
|
+
lastSuccess: AcquisitionRunRecord | null;
|
|
12
|
+
lastRun: AcquisitionRunRecord | null;
|
|
13
|
+
lease: StreamLease | null;
|
|
14
|
+
nowIso?: string;
|
|
15
|
+
}): StreamFreshness;
|
|
16
|
+
//# sourceMappingURL=run-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-store.d.ts","sourceRoot":"","sources":["../../src/connector-ops/run-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAM,WAAW,EAAE,MAAM,SAAS,CAAC;AAG/C,OAAO,EAIL,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EAGxB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAW/B,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,MAAM,GACZ,mBAAmB,CA+GrB;AAED,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,MAAM,GACZ,mBAAmB,CAyIrB;AASD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACzC,OAAO,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACrC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,eAAe,CAoClB"}
|