@lobu/connector-sdk 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-paginated.d.ts +79 -0
- package/dist/api-paginated.d.ts.map +1 -0
- package/dist/api-paginated.js +120 -0
- package/dist/api-paginated.js.map +1 -0
- package/dist/base.d.ts +66 -0
- package/dist/base.d.ts.map +1 -0
- package/dist/base.js +122 -0
- package/dist/base.js.map +1 -0
- package/dist/browser/acquire.d.ts +66 -0
- package/dist/browser/acquire.d.ts.map +1 -0
- package/dist/browser/acquire.js +109 -0
- package/dist/browser/acquire.js.map +1 -0
- package/dist/browser/cdp-page.d.ts +48 -0
- package/dist/browser/cdp-page.d.ts.map +1 -0
- package/dist/browser/cdp-page.js +165 -0
- package/dist/browser/cdp-page.js.map +1 -0
- package/dist/browser/cdp.d.ts +44 -0
- package/dist/browser/cdp.d.ts.map +1 -0
- package/dist/browser/cdp.js +252 -0
- package/dist/browser/cdp.js.map +1 -0
- package/dist/browser/launcher.d.ts +29 -0
- package/dist/browser/launcher.d.ts.map +1 -0
- package/dist/browser/launcher.js +157 -0
- package/dist/browser/launcher.js.map +1 -0
- package/dist/browser/stealth.d.ts +55 -0
- package/dist/browser/stealth.d.ts.map +1 -0
- package/dist/browser/stealth.js +170 -0
- package/dist/browser/stealth.js.map +1 -0
- package/dist/browser-network.d.ts +51 -0
- package/dist/browser-network.d.ts.map +1 -0
- package/dist/browser-network.js +175 -0
- package/dist/browser-network.js.map +1 -0
- package/dist/browser-paginated.d.ts +141 -0
- package/dist/browser-paginated.d.ts.map +1 -0
- package/dist/browser-paginated.js +269 -0
- package/dist/browser-paginated.js.map +1 -0
- package/dist/connector-runtime.d.ts +70 -0
- package/dist/connector-runtime.d.ts.map +1 -0
- package/dist/connector-runtime.js +48 -0
- package/dist/connector-runtime.js.map +1 -0
- package/dist/connector-types.d.ts +613 -0
- package/dist/connector-types.d.ts.map +1 -0
- package/dist/connector-types.js +27 -0
- package/dist/connector-types.js.map +1 -0
- package/dist/event-taxonomy.d.ts +3 -0
- package/dist/event-taxonomy.d.ts.map +1 -0
- package/dist/event-taxonomy.js +30 -0
- package/dist/event-taxonomy.js.map +1 -0
- package/dist/http.d.ts +18 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +74 -0
- package/dist/http.js.map +1 -0
- package/dist/identity-normalize.d.ts +53 -0
- package/dist/identity-normalize.d.ts.map +1 -0
- package/dist/identity-normalize.js +146 -0
- package/dist/identity-normalize.js.map +1 -0
- package/dist/identity-types.d.ts +214 -0
- package/dist/identity-types.d.ts.map +1 -0
- package/dist/identity-types.js +217 -0
- package/dist/identity-types.js.map +1 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +33 -0
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +7 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +10 -0
- package/dist/logger.js.map +1 -0
- package/dist/paginated.d.ts +93 -0
- package/dist/paginated.d.ts.map +1 -0
- package/dist/paginated.js +167 -0
- package/dist/paginated.js.map +1 -0
- package/dist/reaction-sdk.d.ts +43 -0
- package/dist/reaction-sdk.d.ts.map +1 -0
- package/dist/reaction-sdk.js +9 -0
- package/dist/reaction-sdk.js.map +1 -0
- package/dist/retry.d.ts +19 -0
- package/dist/retry.d.ts.map +1 -0
- package/dist/retry.js +131 -0
- package/dist/retry.js.map +1 -0
- package/dist/scoring.d.ts +17 -0
- package/dist/scoring.d.ts.map +1 -0
- package/dist/scoring.js +28 -0
- package/dist/scoring.js.map +1 -0
- package/dist/types.d.ts +280 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/watcher-time.d.ts +14 -0
- package/dist/watcher-time.d.ts.map +1 -0
- package/dist/watcher-time.js +112 -0
- package/dist/watcher-time.js.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Identity-engine SDK contracts.
|
|
3
|
+
*
|
|
4
|
+
* Connectors emit `ConnectorFact[]` for the authenticated subject; catalog
|
|
5
|
+
* YAMLs declare `AutoCreateWhenRule[]` on relationship types; the engine
|
|
6
|
+
* matches facts against rules and writes derivations.
|
|
7
|
+
*
|
|
8
|
+
* Schemas are TypeBox so they double as runtime validators at every write
|
|
9
|
+
* boundary (seeder, engine, MCP tools). Validation is mandatory — the
|
|
10
|
+
* collapsed model puts a lot in `metadata jsonb`, so without validation the
|
|
11
|
+
* engine silently corrupts on malformed input.
|
|
12
|
+
*/
|
|
13
|
+
import { Type } from '@sinclair/typebox';
|
|
14
|
+
// =============================================================================
|
|
15
|
+
// Assurance levels
|
|
16
|
+
// =============================================================================
|
|
17
|
+
/**
|
|
18
|
+
* How strongly the connector vouches for a fact. Rules require a minimum
|
|
19
|
+
* assurance to fire. The order is total: `oauth_verified_admin_role` >
|
|
20
|
+
* `oauth_verified` > `cookie_session` > `self_attested`.
|
|
21
|
+
*/
|
|
22
|
+
export const AssuranceLevel = Type.Union([
|
|
23
|
+
Type.Literal('oauth_verified_admin_role'),
|
|
24
|
+
Type.Literal('oauth_verified'),
|
|
25
|
+
Type.Literal('cookie_session'),
|
|
26
|
+
Type.Literal('self_attested'),
|
|
27
|
+
], { $id: 'AssuranceLevel' });
|
|
28
|
+
const ASSURANCE_RANK = {
|
|
29
|
+
oauth_verified_admin_role: 4,
|
|
30
|
+
oauth_verified: 3,
|
|
31
|
+
cookie_session: 2,
|
|
32
|
+
self_attested: 1,
|
|
33
|
+
};
|
|
34
|
+
export function assuranceMeets(actual, required) {
|
|
35
|
+
return ASSURANCE_RANK[actual] >= ASSURANCE_RANK[required];
|
|
36
|
+
}
|
|
37
|
+
// =============================================================================
|
|
38
|
+
// ConnectorFact — what a connector emits
|
|
39
|
+
// =============================================================================
|
|
40
|
+
export const ConnectorFact = Type.Object({
|
|
41
|
+
/** The attribute kind. Each connector declares which namespaces it produces. */
|
|
42
|
+
namespace: Type.String({ minLength: 1, maxLength: 64 }),
|
|
43
|
+
/** Raw value as the provider returned it. Audit-friendly. */
|
|
44
|
+
identifier: Type.String({ minLength: 1, maxLength: 1024 }),
|
|
45
|
+
/**
|
|
46
|
+
* Canonicalised form used for index lookups. Connectors run normalize* on
|
|
47
|
+
* the value before emitting; the engine re-normalises defensively.
|
|
48
|
+
*/
|
|
49
|
+
normalizedValue: Type.String({ minLength: 1, maxLength: 1024 }),
|
|
50
|
+
/** How the connector verified this fact. */
|
|
51
|
+
assurance: AssuranceLevel,
|
|
52
|
+
/**
|
|
53
|
+
* Provider's immutable account identifier — the primary binding key
|
|
54
|
+
* that survives email changes and account-row reissues.
|
|
55
|
+
*/
|
|
56
|
+
providerStableId: Type.String({ minLength: 1, maxLength: 256 }),
|
|
57
|
+
/**
|
|
58
|
+
* Which connector account row produced this fact. Lets the engine diff
|
|
59
|
+
* facts per session for revocation.
|
|
60
|
+
*/
|
|
61
|
+
sourceAccountId: Type.String({ minLength: 1, maxLength: 256 }),
|
|
62
|
+
/** ISO-8601 timestamp. Optional; required for high-assurance facts in production. */
|
|
63
|
+
validTo: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })),
|
|
64
|
+
/**
|
|
65
|
+
* Optional human-readable note about trust caveats this fact carries.
|
|
66
|
+
* Surfaced in audit logs and admin UIs; never used by the engine.
|
|
67
|
+
*/
|
|
68
|
+
notes: Type.Optional(Type.String({ maxLength: 512 })),
|
|
69
|
+
}, { $id: 'ConnectorFact', additionalProperties: false });
|
|
70
|
+
// =============================================================================
|
|
71
|
+
// ConnectorIdentityCapability — what a connector promises to emit
|
|
72
|
+
// =============================================================================
|
|
73
|
+
/**
|
|
74
|
+
* Each connector exports a static capability declaration. CI lints that the
|
|
75
|
+
* namespaces it actually emits at runtime are a subset of `produces`.
|
|
76
|
+
*/
|
|
77
|
+
export const ConnectorIdentityCapability = Type.Object({
|
|
78
|
+
connectorKey: Type.String({ minLength: 1, maxLength: 64 }),
|
|
79
|
+
produces: Type.Array(Type.Object({
|
|
80
|
+
namespace: Type.String({ minLength: 1, maxLength: 64 }),
|
|
81
|
+
assurance: AssuranceLevel,
|
|
82
|
+
notes: Type.Optional(Type.String({ maxLength: 512 })),
|
|
83
|
+
}), { minItems: 0, maxItems: 32 }),
|
|
84
|
+
}, { $id: 'ConnectorIdentityCapability', additionalProperties: false });
|
|
85
|
+
// =============================================================================
|
|
86
|
+
// AutoCreateWhenRule — declared on relationship-type YAMLs, compiled into
|
|
87
|
+
// entity_relationship_types.metadata.auto_create_when[]
|
|
88
|
+
// =============================================================================
|
|
89
|
+
/**
|
|
90
|
+
* `unique_only` rejects ambiguity (writes a collision event for admin
|
|
91
|
+
* resolution); `all_matches` fans out to every match and is only safe for
|
|
92
|
+
* weak relationships. `first_match` is deliberately excluded — silently
|
|
93
|
+
* picking a winner on collision would open adoption-takeover paths.
|
|
94
|
+
*/
|
|
95
|
+
export const MatchStrategy = Type.Union([Type.Literal('unique_only'), Type.Literal('all_matches')], { $id: 'MatchStrategy' });
|
|
96
|
+
export const AutoCreateWhenRule = Type.Object({
|
|
97
|
+
/** The fact namespace this rule listens for. */
|
|
98
|
+
sourceNamespace: Type.String({ minLength: 1, maxLength: 64 }),
|
|
99
|
+
/**
|
|
100
|
+
* The metadata field on the target entity-type whose normalized value
|
|
101
|
+
* must equal the fact's normalizedValue.
|
|
102
|
+
*/
|
|
103
|
+
targetField: Type.String({ minLength: 1, maxLength: 64 }),
|
|
104
|
+
/**
|
|
105
|
+
* Minimum assurance the fact must carry to fire this rule. The engine
|
|
106
|
+
* enforces `assuranceMeets(fact.assurance, rule.assuranceRequired)`.
|
|
107
|
+
*/
|
|
108
|
+
assuranceRequired: AssuranceLevel,
|
|
109
|
+
matchStrategy: MatchStrategy,
|
|
110
|
+
/**
|
|
111
|
+
* Optional notes for review tooling — not consumed by the engine.
|
|
112
|
+
*/
|
|
113
|
+
notes: Type.Optional(Type.String({ maxLength: 512 })),
|
|
114
|
+
}, { $id: 'AutoCreateWhenRule', additionalProperties: false });
|
|
115
|
+
/**
|
|
116
|
+
* The shape stored on `entity_relationship_types.metadata`. Includes the
|
|
117
|
+
* declared rules plus a version+hash so derivations can pin to the rule
|
|
118
|
+
* version that fired them and reconciliation can detect drift.
|
|
119
|
+
*/
|
|
120
|
+
export const RelationshipTypeIdentityMetadata = Type.Object({
|
|
121
|
+
autoCreateWhen: Type.Array(AutoCreateWhenRule, { maxItems: 16 }),
|
|
122
|
+
/** Monotonically increasing per relationship-type. Bumped by the seeder on every YAML change. */
|
|
123
|
+
ruleVersion: Type.Integer({ minimum: 1 }),
|
|
124
|
+
/** sha256 of the canonicalised auto_create_when array. Drift detection. */
|
|
125
|
+
ruleHash: Type.String({ minLength: 64, maxLength: 64 }),
|
|
126
|
+
}, { $id: 'RelationshipTypeIdentityMetadata', additionalProperties: true });
|
|
127
|
+
// =============================================================================
|
|
128
|
+
// Fact event metadata — written into events.metadata when the engine
|
|
129
|
+
// persists a connector fact as `semantic_type='identity_fact'`
|
|
130
|
+
// =============================================================================
|
|
131
|
+
/**
|
|
132
|
+
* Stored on `events.metadata` for `semantic_type='identity_fact'` rows.
|
|
133
|
+
*
|
|
134
|
+
* Tombstone facts (written when a connector refresh stops emitting a
|
|
135
|
+
* namespace) carry empty `identifier`/`normalizedValue`/`providerStableId`,
|
|
136
|
+
* which is why those fields allow empty strings here even though
|
|
137
|
+
* `ConnectorFact` (the connector-side input) requires non-empty.
|
|
138
|
+
*/
|
|
139
|
+
export const FactEventMetadata = Type.Object({
|
|
140
|
+
namespace: Type.String({ minLength: 1, maxLength: 64 }),
|
|
141
|
+
identifier: Type.String({ minLength: 0, maxLength: 1024 }),
|
|
142
|
+
normalizedValue: Type.String({ minLength: 0, maxLength: 1024 }),
|
|
143
|
+
assurance: AssuranceLevel,
|
|
144
|
+
providerStableId: Type.String({ minLength: 0, maxLength: 256 }),
|
|
145
|
+
sourceAccountId: Type.String({ minLength: 1, maxLength: 256 }),
|
|
146
|
+
/** ISO-8601 timestamp string. Optional; required for high-assurance facts in production. */
|
|
147
|
+
validTo: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })),
|
|
148
|
+
notes: Type.Optional(Type.String({ maxLength: 512 })),
|
|
149
|
+
}, { $id: 'FactEventMetadata', additionalProperties: false });
|
|
150
|
+
// =============================================================================
|
|
151
|
+
// DerivedFromProvenance — written into entity_relationships.metadata when
|
|
152
|
+
// the engine auto-creates a relationship
|
|
153
|
+
// =============================================================================
|
|
154
|
+
export const DerivedFromProvenance = Type.Object({
|
|
155
|
+
/** events.id of the fact that produced this derivation. */
|
|
156
|
+
sourceEventId: Type.Integer({ minimum: 1 }),
|
|
157
|
+
/** Which relationship-type rule fired (entity_relationship_types.id). */
|
|
158
|
+
relationshipTypeId: Type.Integer({ minimum: 1 }),
|
|
159
|
+
/** Snapshot of the rule version that fired. Drift signal for reconcile. */
|
|
160
|
+
ruleVersion: Type.Integer({ minimum: 1 }),
|
|
161
|
+
/** Hash matching ruleHash on the relationship-type at fire time. */
|
|
162
|
+
ruleHash: Type.String({ minLength: 64, maxLength: 64 }),
|
|
163
|
+
/** Echoed for fast assurance audits without joining back to the event. */
|
|
164
|
+
factAssurance: AssuranceLevel,
|
|
165
|
+
/** ISO-8601 timestamp. Echoes events.created_at; convenience for relationship-only audits. */
|
|
166
|
+
derivedAt: Type.String({ minLength: 1, maxLength: 64 }),
|
|
167
|
+
}, { $id: 'DerivedFromProvenance', additionalProperties: false });
|
|
168
|
+
/**
|
|
169
|
+
* The metadata blob stored on entity_relationships.metadata for derived
|
|
170
|
+
* rows. Closed shape — extra keys are rejected so the engine can never
|
|
171
|
+
* accidentally land junk alongside the provenance.
|
|
172
|
+
*/
|
|
173
|
+
export const DerivedRelationshipMetadata = Type.Object({
|
|
174
|
+
derivedFrom: DerivedFromProvenance,
|
|
175
|
+
}, { $id: 'DerivedRelationshipMetadata', additionalProperties: false });
|
|
176
|
+
// =============================================================================
|
|
177
|
+
// ClaimCollision — pending-approval event payload
|
|
178
|
+
// =============================================================================
|
|
179
|
+
/**
|
|
180
|
+
* When a fact match would adopt a `$member` row that's already bound to a
|
|
181
|
+
* different user (or vice versa), the engine writes a pending-approval event
|
|
182
|
+
* with this shape. Resolution = a privileged user flips
|
|
183
|
+
* `interaction_status='approved'` after merging entities, or 'rejected' to
|
|
184
|
+
* dismiss without action.
|
|
185
|
+
*/
|
|
186
|
+
export const ClaimCollisionPayload = Type.Object({
|
|
187
|
+
kind: Type.Literal('identity_match'),
|
|
188
|
+
namespace: Type.String({ minLength: 1, maxLength: 64 }),
|
|
189
|
+
identifier: Type.String({ minLength: 1, maxLength: 1024 }),
|
|
190
|
+
normalizedValue: Type.String({ minLength: 1, maxLength: 1024 }),
|
|
191
|
+
/**
|
|
192
|
+
* Catalog entity ids that all match this fact's normalized value.
|
|
193
|
+
* Despite the historical name suggesting `$member`, these are general
|
|
194
|
+
* catalog entities (founders, companies, etc.) — the resolver picks
|
|
195
|
+
* which one the user intends to claim.
|
|
196
|
+
*/
|
|
197
|
+
candidateEntityIds: Type.Array(Type.Integer({ minimum: 1 }), {
|
|
198
|
+
minItems: 2,
|
|
199
|
+
maxItems: 16,
|
|
200
|
+
}),
|
|
201
|
+
/** Total number of matching candidates; candidateEntityIds may be capped for UI payload size. */
|
|
202
|
+
candidateCount: Type.Optional(Type.Integer({ minimum: 2 })),
|
|
203
|
+
/**
|
|
204
|
+
* Which fact (events.id) raised the collision. Lets the resolver replay
|
|
205
|
+
* the binding once the human picks a winner.
|
|
206
|
+
*/
|
|
207
|
+
triggeringEventId: Type.Integer({ minimum: 1 }),
|
|
208
|
+
/** Rule that would have fired if the match were unambiguous. */
|
|
209
|
+
relationshipTypeId: Type.Optional(Type.Integer({ minimum: 1 })),
|
|
210
|
+
}, { $id: 'ClaimCollisionPayload', additionalProperties: false });
|
|
211
|
+
// =============================================================================
|
|
212
|
+
// Constants — used by core code AND connector tests so the lint check has
|
|
213
|
+
// a single source of truth.
|
|
214
|
+
// =============================================================================
|
|
215
|
+
export const IDENTITY_FACT_SEMANTIC_TYPE = 'identity_fact';
|
|
216
|
+
export const CLAIM_COLLISION_SEMANTIC_TYPE = 'claim_collision';
|
|
217
|
+
//# sourceMappingURL=identity-types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"identity-types.js","sourceRoot":"","sources":["../src/identity-types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,IAAI,EAAe,MAAM,mBAAmB,CAAC;AAEtD,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CACtC;IACE,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAC9B,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IAC9B,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC;CAC9B,EACD,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAC1B,CAAC;AAGF,MAAM,cAAc,GAAmC;IACrD,yBAAyB,EAAE,CAAC;IAC5B,cAAc,EAAE,CAAC;IACjB,cAAc,EAAE,CAAC;IACjB,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,MAAsB,EAAE,QAAwB;IAC7E,OAAO,cAAc,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;AAC5D,CAAC;AAED,gFAAgF;AAChF,yCAAyC;AACzC,gFAAgF;AAEhF,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CACtC;IACE,gFAAgF;IAChF,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACvD,6DAA6D;IAC7D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1D;;;OAGG;IACH,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/D,4CAA4C;IAC5C,SAAS,EAAE,cAAc;IACzB;;;OAGG;IACH,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IAC/D;;;OAGG;IACH,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IAC9D,qFAAqF;IACrF,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IACpE;;;OAGG;IACH,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;CACtD,EACD,EAAE,GAAG,EAAE,eAAe,EAAE,oBAAoB,EAAE,KAAK,EAAE,CACtD,CAAC;AAGF,gFAAgF;AAChF,kEAAkE;AAClE,gFAAgF;AAEhF;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,MAAM,CACpD;IACE,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC1D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAClB,IAAI,CAAC,MAAM,CAAC;QACV,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QACvD,SAAS,EAAE,cAAc;QACzB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;KACtD,CAAC,EACF,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC9B;CACF,EACD,EAAE,GAAG,EAAE,6BAA6B,EAAE,oBAAoB,EAAE,KAAK,EAAE,CACpE,CAAC;AAGF,gFAAgF;AAChF,0EAA0E;AAC1E,wDAAwD;AACxD,gFAAgF;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CACrC,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,EAC1D,EAAE,GAAG,EAAE,eAAe,EAAE,CACzB,CAAC;AAGF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAC3C;IACE,gDAAgD;IAChD,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC7D;;;OAGG;IACH,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACzD;;;OAGG;IACH,iBAAiB,EAAE,cAAc;IACjC,aAAa,EAAE,aAAa;IAC5B;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;CACtD,EACD,EAAE,GAAG,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC3D,CAAC;AAGF;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC,MAAM,CACzD;IACE,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAChE,iGAAiG;IACjG,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACzC,2EAA2E;IAC3E,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;CACxD,EACD,EAAE,GAAG,EAAE,kCAAkC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CACxE,CAAC;AAGF,gFAAgF;AAChF,qEAAqE;AACrE,+DAA+D;AAC/D,gFAAgF;AAEhF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAC1C;IACE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1D,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/D,SAAS,EAAE,cAAc;IACzB,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IAC/D,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IAC9D,4FAA4F;IAC5F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IACpE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;CACtD,EACD,EAAE,GAAG,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC1D,CAAC;AAGF,gFAAgF;AAChF,0EAA0E;AAC1E,yCAAyC;AACzC,gFAAgF;AAEhF,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,MAAM,CAC9C;IACE,2DAA2D;IAC3D,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC3C,yEAAyE;IACzE,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAChD,2EAA2E;IAC3E,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACzC,oEAAoE;IACpE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACvD,0EAA0E;IAC1E,aAAa,EAAE,cAAc;IAC7B,8FAA8F;IAC9F,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;CACxD,EACD,EAAE,GAAG,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC9D,CAAC;AAGF;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,MAAM,CACpD;IACE,WAAW,EAAE,qBAAqB;CACnC,EACD,EAAE,GAAG,EAAE,6BAA6B,EAAE,oBAAoB,EAAE,KAAK,EAAE,CACpE,CAAC;AAGF,gFAAgF;AAChF,kDAAkD;AAClD,gFAAgF;AAEhF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,MAAM,CAC9C;IACE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACpC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC1D,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/D;;;;;OAKG;IACH,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE;QAC3D,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,EAAE;KACb,CAAC;IACF,iGAAiG;IACjG,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3D;;;OAGG;IACH,iBAAiB,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/C,gEAAgE;IAChE,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;CAChE,EACD,EAAE,GAAG,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAC9D,CAAC;AAGF,gFAAgF;AAChF,0EAA0E;AAC1E,4BAA4B;AAC5B,gFAAgF;AAEhF,MAAM,CAAC,MAAM,2BAA2B,GAAG,eAAwB,CAAC;AACpE,MAAM,CAAC,MAAM,6BAA6B,GAAG,iBAA0B,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export type { Static } from '@sinclair/typebox';
|
|
2
|
+
export { Type } from '@sinclair/typebox';
|
|
3
|
+
export type { KyInstance, Options } from 'ky';
|
|
4
|
+
export { default as ky, HTTPError } from 'ky';
|
|
5
|
+
export { ConnectorRuntime } from './connector-runtime.js';
|
|
6
|
+
export type { ActionContext, ActionDefinition, ActionResult, ApprovalStatus, AuthArtifact, AuthContext, AuthResult, Connection, ConnectorAuthBrowser, ConnectorAuthEnvField, ConnectorAuthEnvKeys, ConnectorAuthInteractive, ConnectorAuthMethod, ConnectorAuthNone, ConnectorAuthOAuth, ConnectorAuthSchema, ConnectorDefinition, ContentItem, EntityIdentitySpec, EntityLinkOverride, EntityLinkOverrides, EntityLinkRule, EntityTraitSpec, EventEnvelope, Feed, FeedDefinition, FeedMode, IdentityNamespace, Run, RunStatus, RunType, SyncContext, SyncCredentials, SyncResult, } from './connector-types.js';
|
|
7
|
+
export { IDENTITY } from './connector-types.js';
|
|
8
|
+
export { AssuranceLevel, assuranceMeets, AutoCreateWhenRule, CLAIM_COLLISION_SEMANTIC_TYPE, ClaimCollisionPayload, ConnectorFact, ConnectorIdentityCapability, DerivedFromProvenance, DerivedRelationshipMetadata, FactEventMetadata, IDENTITY_FACT_SEMANTIC_TYPE, MatchStrategy, RelationshipTypeIdentityMetadata, } from './identity-types.js';
|
|
9
|
+
export { isSourceNativeEventType, SOURCE_NATIVE_EVENT_TYPES } from './event-taxonomy.js';
|
|
10
|
+
export { createAuthenticatedClient, createHttpClient, httpClient, jsonHttpClient, } from './http.js';
|
|
11
|
+
export { normalizeAuthUserId, normalizeEmail, normalizeGithubLogin, normalizeGoogleContactId, normalizeIdentifier, normalizePhone, normalizeSlackUserId, normalizeWaJid, } from './identity-normalize.js';
|
|
12
|
+
export { sdkLogger, sdkLogger as logger } from './logger.js';
|
|
13
|
+
export { withHttpRetry } from './retry.js';
|
|
14
|
+
export { calculateEngagementScore } from './scoring.js';
|
|
15
|
+
export type { WatcherTimeGranularity } from './watcher-time.js';
|
|
16
|
+
export { addWatcherPeriod, alignToWatcherWindowStart, getAvailableWatcherGranularities, getFinerWatcherGranularities, getNextWatcherGranularity, getWatcherDateTruncUnit, inferWatcherGranularityFromDays, inferWatcherGranularityFromSchedule, isWatcherTimeGranularity, shiftWatcherPeriod, subtractWatcherPeriod, WATCHER_TIME_GRANULARITIES, } from './watcher-time.js';
|
|
17
|
+
export type { ApiSessionState } from './api-paginated.js';
|
|
18
|
+
export { ApiPaginatedFeed } from './api-paginated.js';
|
|
19
|
+
export { BaseFeed, RateLimitError } from './base.js';
|
|
20
|
+
export type { AcquireBrowserOptions, AcquiredBrowser } from './browser/acquire.js';
|
|
21
|
+
export { acquireBrowser, BrowserAuthCascadeError } from './browser/acquire.js';
|
|
22
|
+
export type { CdpVersionInfo, ResolveCdpOptions } from './browser/cdp.js';
|
|
23
|
+
export { discoverChromeListeningPorts, discoverChromeProcessCdpUrls, fetchCdpVersionInfo, normalizeCdpUrl, resolveCdpUrl, tryWebSocketCdp, } from './browser/cdp.js';
|
|
24
|
+
export { CdpPage } from './browser/cdp-page.js';
|
|
25
|
+
export type { BrowserLaunchOptions, EnhancedBrowser } from './browser/launcher.js';
|
|
26
|
+
export { captureErrorArtifacts, launchBrowser, withErrorCapture, } from './browser/launcher.js';
|
|
27
|
+
export type { StealthBrowser, StealthBrowserOptions } from './browser/stealth.js';
|
|
28
|
+
export { getRandomDelay, humanWait, launchStealthBrowser, randomScroll, testBotDetection, } from './browser/stealth.js';
|
|
29
|
+
export type { BrowserNetworkConfig, BrowserNetworkResult } from './browser-network.js';
|
|
30
|
+
export { browserNetworkSync } from './browser-network.js';
|
|
31
|
+
export type { BrowserFeedConfig, BrowserPaginationConfig, BrowserSessionState, CaptchaConfig, CookieConsentConfig, } from './browser-paginated.js';
|
|
32
|
+
export { BrowserPaginatedFeed } from './browser-paginated.js';
|
|
33
|
+
export type { PageFetchResult, PaginatedCheckpoint, PaginateResult, PaginationConfig, } from './paginated.js';
|
|
34
|
+
export { PaginatedFeed } from './paginated.js';
|
|
35
|
+
export type { ReactionContext, ReactionEntity } from './reaction-sdk.js';
|
|
36
|
+
export type { Checkpoint, Content, Env, FeedAuthBrowserMethod, FeedAuthEnvField, FeedAuthEnvKeysMethod, FeedAuthMethod, FeedAuthNoneMethod, FeedAuthOAuthMethod, FeedAuthSchema, FeedOptions, FeedSyncResult, IFeed, ParentFeedDefinition, ScoringConfig, SearchResult, SessionState, } from './types.js';
|
|
37
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,YAAY,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAEzC,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,UAAU,EACV,UAAU,EACV,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,eAAe,EACf,aAAa,EACb,IAAI,EACJ,cAAc,EACd,QAAQ,EACR,iBAAiB,EACjB,GAAG,EACH,SAAS,EACT,OAAO,EACP,WAAW,EACX,eAAe,EACf,UAAU,GACX,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EACL,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,6BAA6B,EAC7B,qBAAqB,EACrB,aAAa,EACb,2BAA2B,EAC3B,qBAAqB,EACrB,2BAA2B,EAC3B,iBAAiB,EACjB,2BAA2B,EAC3B,aAAa,EACb,gCAAgC,GACjC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,UAAU,EACV,cAAc,GACf,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,cAAc,GACf,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAC;AACxD,YAAY,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EACL,gBAAgB,EAChB,yBAAyB,EACzB,gCAAgC,EAChC,4BAA4B,EAC5B,yBAAyB,EACzB,uBAAuB,EACvB,+BAA+B,EAC/B,mCAAmC,EACnC,wBAAwB,EACxB,kBAAkB,EAClB,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,mBAAmB,CAAC;AAM3B,YAAY,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,YAAY,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EACL,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClF,OAAO,EACL,cAAc,EACd,SAAS,EACT,oBAAoB,EACpB,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACvF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,YAAY,EACV,iBAAiB,EACjB,uBAAuB,EACvB,mBAAmB,EACnB,aAAa,EACb,mBAAmB,GACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,YAAY,EACV,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACzE,YAAY,EACV,UAAU,EACV,OAAO,EACP,GAAG,EACH,qBAAqB,EACrB,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,EACd,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,EACd,WAAW,EACX,cAAc,EACd,KAAK,EACL,oBAAoB,EACpB,aAAa,EACb,YAAY,EACZ,YAAY,GACb,MAAM,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// V1 Integration Platform — Connector SDK
|
|
3
|
+
// =============================================================================
|
|
4
|
+
export { Type } from '@sinclair/typebox';
|
|
5
|
+
export { default as ky, HTTPError } from 'ky';
|
|
6
|
+
// Connector runtime & types (primary API)
|
|
7
|
+
export { ConnectorRuntime } from './connector-runtime.js';
|
|
8
|
+
export { IDENTITY } from './connector-types.js';
|
|
9
|
+
// Identity-engine SDK contracts. Each schema export is both a TypeBox
|
|
10
|
+
// runtime validator (value) AND a TypeScript type via declaration merging.
|
|
11
|
+
export { AssuranceLevel, assuranceMeets, AutoCreateWhenRule, CLAIM_COLLISION_SEMANTIC_TYPE, ClaimCollisionPayload, ConnectorFact, ConnectorIdentityCapability, DerivedFromProvenance, DerivedRelationshipMetadata, FactEventMetadata, IDENTITY_FACT_SEMANTIC_TYPE, MatchStrategy, RelationshipTypeIdentityMetadata, } from './identity-types.js';
|
|
12
|
+
export { isSourceNativeEventType, SOURCE_NATIVE_EVENT_TYPES } from './event-taxonomy.js';
|
|
13
|
+
// HTTP clients
|
|
14
|
+
export { createAuthenticatedClient, createHttpClient, httpClient, jsonHttpClient, } from './http.js';
|
|
15
|
+
export { normalizeAuthUserId, normalizeEmail, normalizeGithubLogin, normalizeGoogleContactId, normalizeIdentifier, normalizePhone, normalizeSlackUserId, normalizeWaJid, } from './identity-normalize.js';
|
|
16
|
+
// Logger
|
|
17
|
+
export { sdkLogger, sdkLogger as logger } from './logger.js';
|
|
18
|
+
// Retry
|
|
19
|
+
export { withHttpRetry } from './retry.js';
|
|
20
|
+
// Scoring
|
|
21
|
+
export { calculateEngagementScore } from './scoring.js';
|
|
22
|
+
export { addWatcherPeriod, alignToWatcherWindowStart, getAvailableWatcherGranularities, getFinerWatcherGranularities, getNextWatcherGranularity, getWatcherDateTruncUnit, inferWatcherGranularityFromDays, inferWatcherGranularityFromSchedule, isWatcherTimeGranularity, shiftWatcherPeriod, subtractWatcherPeriod, WATCHER_TIME_GRANULARITIES, } from './watcher-time.js';
|
|
23
|
+
export { ApiPaginatedFeed } from './api-paginated.js';
|
|
24
|
+
export { BaseFeed, RateLimitError } from './base.js';
|
|
25
|
+
export { acquireBrowser, BrowserAuthCascadeError } from './browser/acquire.js';
|
|
26
|
+
export { discoverChromeListeningPorts, discoverChromeProcessCdpUrls, fetchCdpVersionInfo, normalizeCdpUrl, resolveCdpUrl, tryWebSocketCdp, } from './browser/cdp.js';
|
|
27
|
+
export { CdpPage } from './browser/cdp-page.js';
|
|
28
|
+
export { captureErrorArtifacts, launchBrowser, withErrorCapture, } from './browser/launcher.js';
|
|
29
|
+
export { getRandomDelay, humanWait, launchStealthBrowser, randomScroll, testBotDetection, } from './browser/stealth.js';
|
|
30
|
+
export { browserNetworkSync } from './browser-network.js';
|
|
31
|
+
export { BrowserPaginatedFeed } from './browser-paginated.js';
|
|
32
|
+
export { PaginatedFeed } from './paginated.js';
|
|
33
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,0CAA0C;AAC1C,gFAAgF;AAIhF,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAGzC,OAAO,EAAE,OAAO,IAAI,EAAE,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC9C,0CAA0C;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAqC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,sEAAsE;AACtE,2EAA2E;AAC3E,OAAO,EACL,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,6BAA6B,EAC7B,qBAAqB,EACrB,aAAa,EACb,2BAA2B,EAC3B,qBAAqB,EACrB,2BAA2B,EAC3B,iBAAiB,EACjB,2BAA2B,EAC3B,aAAa,EACb,gCAAgC,GACjC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACzF,eAAe;AACf,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,UAAU,EACV,cAAc,GACf,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,wBAAwB,EACxB,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,SAAS;AACT,OAAO,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,MAAM,aAAa,CAAC;AAC7D,QAAQ;AACR,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,UAAU;AACV,OAAO,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EACL,gBAAgB,EAChB,yBAAyB,EACzB,gCAAgC,EAChC,4BAA4B,EAC5B,yBAAyB,EACzB,uBAAuB,EACvB,+BAA+B,EAC/B,mCAAmC,EACnC,wBAAwB,EACxB,kBAAkB,EAClB,qBAAqB,EACrB,0BAA0B,GAC3B,MAAM,mBAAmB,CAAC;AAO3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EACL,4BAA4B,EAC5B,4BAA4B,EAC5B,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAEhD,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,cAAc,EACd,SAAS,EACT,oBAAoB,EACpB,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAQ1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAO9D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
package/dist/logger.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,SAAS,6BAGpB,CAAC"}
|
package/dist/logger.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import pino from 'pino';
|
|
2
|
+
/**
|
|
3
|
+
* SDK logger instance
|
|
4
|
+
* Uses its own pino instance, separate from the main app logger
|
|
5
|
+
*/
|
|
6
|
+
export const sdkLogger = pino({
|
|
7
|
+
name: 'connector-sdk',
|
|
8
|
+
level: process.env.LOG_LEVEL || 'info',
|
|
9
|
+
});
|
|
10
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,eAAe;IACrB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;CACvC,CAAC,CAAC"}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Paginated Feed Base Class
|
|
3
|
+
*
|
|
4
|
+
* Provides a reusable pagination pattern for feeds that fetch data in pages.
|
|
5
|
+
* Uses the template method pattern - subclasses implement specific methods while
|
|
6
|
+
* the base class handles the pagination loop, rate limiting, and checkpoint management.
|
|
7
|
+
*/
|
|
8
|
+
import { BaseFeed } from './base.js';
|
|
9
|
+
import type { Checkpoint, Content, Env, FeedOptions } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Configuration for pagination behavior
|
|
12
|
+
*/
|
|
13
|
+
export interface PaginationConfig {
|
|
14
|
+
/** Maximum pages to fetch (safety limit). Default: 50 */
|
|
15
|
+
maxPages: number;
|
|
16
|
+
/** Expected items per page (for partial page detection). Default: 100 */
|
|
17
|
+
pageSize: number;
|
|
18
|
+
/** Milliseconds to wait between page requests. Default: 1000 */
|
|
19
|
+
rateLimitMs: number;
|
|
20
|
+
/** Whether to update checkpoint after each page (for resume). Default: false */
|
|
21
|
+
incrementalCheckpoint: boolean;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Result from fetching a single page
|
|
25
|
+
*/
|
|
26
|
+
export interface PageFetchResult<TItem> {
|
|
27
|
+
/** Items from this page */
|
|
28
|
+
items: TItem[];
|
|
29
|
+
/** Token/cursor for next page (null = no more pages) */
|
|
30
|
+
nextToken: string | null;
|
|
31
|
+
/** Optional: raw item count before any filtering (for logging) */
|
|
32
|
+
rawCount?: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Extended checkpoint interface with pagination token
|
|
36
|
+
*/
|
|
37
|
+
export interface PaginatedCheckpoint extends Checkpoint {
|
|
38
|
+
/** Pagination token for resuming sync */
|
|
39
|
+
pagination_token?: string | null;
|
|
40
|
+
/** Whether initial sync completed (used to choose checkpoint boundary) */
|
|
41
|
+
initial_complete?: boolean;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Result from the paginate() method
|
|
45
|
+
*/
|
|
46
|
+
export interface PaginateResult<TCheckpoint extends Checkpoint> {
|
|
47
|
+
contents: Content[];
|
|
48
|
+
checkpoint: TCheckpoint;
|
|
49
|
+
parentMap?: Map<string, string>;
|
|
50
|
+
nextSyncRecommendedAt?: Date;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Base class for feeds with cursor/page-based pagination
|
|
54
|
+
*/
|
|
55
|
+
export declare abstract class PaginatedFeed<TItem, TCheckpoint extends PaginatedCheckpoint = PaginatedCheckpoint> extends BaseFeed {
|
|
56
|
+
/**
|
|
57
|
+
* Default pagination configuration
|
|
58
|
+
*/
|
|
59
|
+
protected getPaginationConfig(): PaginationConfig;
|
|
60
|
+
/**
|
|
61
|
+
* ABSTRACT: Fetch a single page of items from the platform
|
|
62
|
+
*/
|
|
63
|
+
protected abstract fetchPage(cursor: string | null, options: FeedOptions, env: Env): Promise<PageFetchResult<TItem>>;
|
|
64
|
+
/**
|
|
65
|
+
* ABSTRACT: Transform a platform item to Content format
|
|
66
|
+
*/
|
|
67
|
+
protected abstract transformItem(item: TItem, options: FeedOptions): Content;
|
|
68
|
+
/**
|
|
69
|
+
* ABSTRACT: Extract published date from platform item
|
|
70
|
+
*/
|
|
71
|
+
protected abstract getItemDate(item: TItem): Date;
|
|
72
|
+
/**
|
|
73
|
+
* OPTIONAL: Filter items before transformation
|
|
74
|
+
*/
|
|
75
|
+
protected filterItem(_item: TItem, _options: FeedOptions): boolean;
|
|
76
|
+
/**
|
|
77
|
+
* OPTIONAL: Extract parent ID for hierarchical content
|
|
78
|
+
*/
|
|
79
|
+
protected getParentId(_item: TItem): string | null;
|
|
80
|
+
/**
|
|
81
|
+
* OPTIONAL: Extract pagination token from checkpoint
|
|
82
|
+
*/
|
|
83
|
+
protected getPaginationToken(checkpoint: TCheckpoint | null): string | null;
|
|
84
|
+
/**
|
|
85
|
+
* OPTIONAL: Create checkpoint with feed-specific fields
|
|
86
|
+
*/
|
|
87
|
+
protected createCheckpoint(existing: TCheckpoint | null, latestContent: Content | null, nextToken: string | null, itemsProcessed: number): TCheckpoint;
|
|
88
|
+
/**
|
|
89
|
+
* Main pagination loop - handles both initial and incremental syncs
|
|
90
|
+
*/
|
|
91
|
+
protected paginate(options: FeedOptions, checkpoint: TCheckpoint | null, env: Env, updateCheckpointFn?: (checkpoint: Checkpoint) => Promise<void>): Promise<PaginateResult<TCheckpoint>>;
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=paginated.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paginated.d.ts","sourceRoot":"","sources":["../src/paginated.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAkB,MAAM,WAAW,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,KAAK;IACpC,2BAA2B;IAC3B,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,wDAAwD;IACxD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,UAAU;IACrD,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,WAAW,SAAS,UAAU;IAC5D,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,UAAU,EAAE,WAAW,CAAC;IACxB,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,qBAAqB,CAAC,EAAE,IAAI,CAAC;CAC9B;AAED;;GAEG;AACH,8BAAsB,aAAa,CACjC,KAAK,EACL,WAAW,SAAS,mBAAmB,GAAG,mBAAmB,CAC7D,SAAQ,QAAQ;IAChB;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,gBAAgB;IASjD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,SAAS,CAC1B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,GAAG,GACP,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAElC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO;IAE5E;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI;IAEjD;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO;IAIlE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,IAAI;IAIlD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,UAAU,EAAE,WAAW,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI;IAI3E;;OAEG;IACH,SAAS,CAAC,gBAAgB,CACxB,QAAQ,EAAE,WAAW,GAAG,IAAI,EAC5B,aAAa,EAAE,OAAO,GAAG,IAAI,EAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,cAAc,EAAE,MAAM,GACrB,WAAW;IAUd;;OAEG;cACa,QAAQ,CACtB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,WAAW,GAAG,IAAI,EAC9B,GAAG,EAAE,GAAG,EACR,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAC7D,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;CA6JxC"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Paginated Feed Base Class
|
|
3
|
+
*
|
|
4
|
+
* Provides a reusable pagination pattern for feeds that fetch data in pages.
|
|
5
|
+
* Uses the template method pattern - subclasses implement specific methods while
|
|
6
|
+
* the base class handles the pagination loop, rate limiting, and checkpoint management.
|
|
7
|
+
*/
|
|
8
|
+
import { BaseFeed, RateLimitError } from './base.js';
|
|
9
|
+
import { sdkLogger } from './logger.js';
|
|
10
|
+
/**
|
|
11
|
+
* Base class for feeds with cursor/page-based pagination
|
|
12
|
+
*/
|
|
13
|
+
export class PaginatedFeed extends BaseFeed {
|
|
14
|
+
/**
|
|
15
|
+
* Default pagination configuration
|
|
16
|
+
*/
|
|
17
|
+
getPaginationConfig() {
|
|
18
|
+
return {
|
|
19
|
+
maxPages: 50,
|
|
20
|
+
pageSize: 100,
|
|
21
|
+
rateLimitMs: 1000,
|
|
22
|
+
incrementalCheckpoint: false,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* OPTIONAL: Filter items before transformation
|
|
27
|
+
*/
|
|
28
|
+
filterItem(_item, _options) {
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* OPTIONAL: Extract parent ID for hierarchical content
|
|
33
|
+
*/
|
|
34
|
+
getParentId(_item) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* OPTIONAL: Extract pagination token from checkpoint
|
|
39
|
+
*/
|
|
40
|
+
getPaginationToken(checkpoint) {
|
|
41
|
+
return checkpoint?.pagination_token ?? null;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* OPTIONAL: Create checkpoint with feed-specific fields
|
|
45
|
+
*/
|
|
46
|
+
createCheckpoint(existing, latestContent, nextToken, itemsProcessed) {
|
|
47
|
+
return {
|
|
48
|
+
updated_at: new Date(),
|
|
49
|
+
last_timestamp: latestContent?.occurred_at ?? existing?.last_timestamp,
|
|
50
|
+
pagination_token: nextToken,
|
|
51
|
+
total_items_processed: (existing?.total_items_processed || 0) + itemsProcessed,
|
|
52
|
+
initial_complete: existing?.initial_complete ?? false,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Main pagination loop - handles both initial and incremental syncs
|
|
57
|
+
*/
|
|
58
|
+
async paginate(options, checkpoint, env, updateCheckpointFn) {
|
|
59
|
+
const config = this.getPaginationConfig();
|
|
60
|
+
const existingToken = this.getPaginationToken(checkpoint);
|
|
61
|
+
const parentMap = new Map();
|
|
62
|
+
const initialComplete = checkpoint?.initial_complete ?? (!!checkpoint?.last_timestamp && !existingToken);
|
|
63
|
+
const isIncremental = initialComplete && !existingToken;
|
|
64
|
+
const lookbackDate = this.getLookbackDate(options);
|
|
65
|
+
const checkpointTimestamp = checkpoint?.last_timestamp ?? null;
|
|
66
|
+
const checkpointDate = checkpointTimestamp
|
|
67
|
+
? checkpointTimestamp instanceof Date
|
|
68
|
+
? checkpointTimestamp
|
|
69
|
+
: new Date(checkpointTimestamp)
|
|
70
|
+
: null;
|
|
71
|
+
const boundaryDate = initialComplete && checkpointDate ? checkpointDate : lookbackDate;
|
|
72
|
+
const allContents = [];
|
|
73
|
+
let cursor = isIncremental ? null : existingToken;
|
|
74
|
+
let pageCount = 0;
|
|
75
|
+
let stopReason = null;
|
|
76
|
+
let rateLimitRetryMs;
|
|
77
|
+
const modeLabel = isIncremental ? 'incremental' : cursor ? 'resumed' : 'initial';
|
|
78
|
+
sdkLogger.info(`[${this.type}] Starting ${modeLabel} sync (max ${config.maxPages} pages)`);
|
|
79
|
+
while (pageCount < config.maxPages) {
|
|
80
|
+
if (pageCount > 0) {
|
|
81
|
+
await this.sleep(config.rateLimitMs);
|
|
82
|
+
}
|
|
83
|
+
let result;
|
|
84
|
+
try {
|
|
85
|
+
result = await this.fetchPage(cursor, options, env);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
if (error instanceof RateLimitError) {
|
|
89
|
+
sdkLogger.warn({ error: error.message }, `[${this.type}] Rate limit hit after ${pageCount} pages, pausing`);
|
|
90
|
+
stopReason = 'rate_limited';
|
|
91
|
+
rateLimitRetryMs = error.retryAfterMs;
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
const rawCount = result.rawCount ?? result.items.length;
|
|
97
|
+
if (result.items.length === 0) {
|
|
98
|
+
sdkLogger.info(`[${this.type}] Empty page at ${pageCount + 1}, stopping`);
|
|
99
|
+
stopReason = 'empty_page';
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
const pageContents = [];
|
|
103
|
+
for (const item of result.items) {
|
|
104
|
+
if (!this.filterItem(item, options))
|
|
105
|
+
continue;
|
|
106
|
+
const content = this.transformItem(item, options);
|
|
107
|
+
pageContents.push(content);
|
|
108
|
+
const parentId = this.getParentId(item);
|
|
109
|
+
if (parentId) {
|
|
110
|
+
parentMap.set(content.origin_id, parentId);
|
|
111
|
+
if (!content.origin_parent_id) {
|
|
112
|
+
content.origin_parent_id = parentId;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const boundaryContents = pageContents.filter((c) => c.occurred_at >= boundaryDate);
|
|
117
|
+
allContents.push(...boundaryContents);
|
|
118
|
+
pageCount++;
|
|
119
|
+
sdkLogger.info(`[${this.type}] Page ${pageCount}: raw=${rawCount}, ` +
|
|
120
|
+
`filtered=${pageContents.length}, recent=${boundaryContents.length}, ` +
|
|
121
|
+
`nextToken=${result.nextToken ? 'yes' : 'no'}`);
|
|
122
|
+
if (config.incrementalCheckpoint && updateCheckpointFn && allContents.length > 0) {
|
|
123
|
+
const incrementalCP = this.createCheckpoint(checkpoint, allContents[0], result.nextToken, boundaryContents.length);
|
|
124
|
+
await updateCheckpointFn(incrementalCP);
|
|
125
|
+
sdkLogger.debug(`[${this.type}] Saved incremental checkpoint after page ${pageCount}`);
|
|
126
|
+
}
|
|
127
|
+
if (boundaryContents.length === 0 && pageContents.length > 0) {
|
|
128
|
+
sdkLogger.info(`[${this.type}] Reached boundary after ${pageCount} pages`);
|
|
129
|
+
stopReason = 'boundary';
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
if (result.items.length < config.pageSize) {
|
|
133
|
+
sdkLogger.info(`[${this.type}] Partial page (${result.items.length}/${config.pageSize}), end reached`);
|
|
134
|
+
stopReason = 'partial_page';
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
if (!result.nextToken) {
|
|
138
|
+
sdkLogger.info(`[${this.type}] No next token after page ${pageCount}, end reached`);
|
|
139
|
+
stopReason = 'no_next_token';
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
cursor = result.nextToken;
|
|
143
|
+
}
|
|
144
|
+
if (!stopReason && pageCount >= config.maxPages) {
|
|
145
|
+
stopReason = 'max_pages';
|
|
146
|
+
}
|
|
147
|
+
if (stopReason === 'max_pages') {
|
|
148
|
+
sdkLogger.warn(`[${this.type}] Hit max page limit (${config.maxPages})`);
|
|
149
|
+
}
|
|
150
|
+
sdkLogger.info(`[${this.type}] Pagination complete: ${allContents.length} items from ${pageCount} pages`);
|
|
151
|
+
const resumeToken = stopReason === 'max_pages' || stopReason === 'rate_limited' ? cursor : null;
|
|
152
|
+
const shouldMarkComplete = initialComplete ||
|
|
153
|
+
(!initialComplete && stopReason !== 'max_pages' && stopReason !== 'rate_limited');
|
|
154
|
+
const nextSyncRecommendedAt = stopReason === 'rate_limited'
|
|
155
|
+
? new Date(Date.now() + (rateLimitRetryMs ?? config.rateLimitMs))
|
|
156
|
+
: undefined;
|
|
157
|
+
const finalCheckpoint = this.createCheckpoint(checkpoint, allContents[0] || null, resumeToken, allContents.length);
|
|
158
|
+
finalCheckpoint.initial_complete = shouldMarkComplete;
|
|
159
|
+
return {
|
|
160
|
+
contents: allContents,
|
|
161
|
+
checkpoint: finalCheckpoint,
|
|
162
|
+
parentMap: parentMap.size > 0 ? parentMap : undefined,
|
|
163
|
+
nextSyncRecommendedAt,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=paginated.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paginated.js","sourceRoot":"","sources":["../src/paginated.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAiDxC;;GAEG;AACH,MAAM,OAAgB,aAGpB,SAAQ,QAAQ;IAChB;;OAEG;IACO,mBAAmB;QAC3B,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,GAAG;YACb,WAAW,EAAE,IAAI;YACjB,qBAAqB,EAAE,KAAK;SAC7B,CAAC;IACJ,CAAC;IAqBD;;OAEG;IACO,UAAU,CAAC,KAAY,EAAE,QAAqB;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,KAAY;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,UAA8B;QACzD,OAAO,UAAU,EAAE,gBAAgB,IAAI,IAAI,CAAC;IAC9C,CAAC;IAED;;OAEG;IACO,gBAAgB,CACxB,QAA4B,EAC5B,aAA6B,EAC7B,SAAwB,EACxB,cAAsB;QAEtB,OAAO;YACL,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,cAAc,EAAE,aAAa,EAAE,WAAW,IAAI,QAAQ,EAAE,cAAc;YACtE,gBAAgB,EAAE,SAAS;YAC3B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,CAAC,CAAC,GAAG,cAAc;YAC9E,gBAAgB,EAAE,QAAQ,EAAE,gBAAgB,IAAI,KAAK;SACvC,CAAC;IACnB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ,CACtB,OAAoB,EACpB,UAA8B,EAC9B,GAAQ,EACR,kBAA8D;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,eAAe,GACnB,UAAU,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,cAAc,IAAI,CAAC,aAAa,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,eAAe,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,mBAAmB,GAAG,UAAU,EAAE,cAAc,IAAI,IAAI,CAAC;QAC/D,MAAM,cAAc,GAAG,mBAAmB;YACxC,CAAC,CAAC,mBAAmB,YAAY,IAAI;gBACnC,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC,CAAC,IAAI,CAAC;QACT,MAAM,YAAY,GAAG,eAAe,IAAI,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;QAEvF,MAAM,WAAW,GAAc,EAAE,CAAC;QAClC,IAAI,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;QAClD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,GAOH,IAAI,CAAC;QAChB,IAAI,gBAAoC,CAAC;QACzC,MAAM,SAAS,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAEjF,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,cAAc,SAAS,cAAc,MAAM,CAAC,QAAQ,SAAS,CAAC,CAAC;QAE3F,OAAO,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,MAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;oBACpC,SAAS,CAAC,IAAI,CACZ,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EACxB,IAAI,IAAI,CAAC,IAAI,0BAA0B,SAAS,iBAAiB,CAClE,CAAC;oBACF,UAAU,GAAG,cAAc,CAAC;oBAC5B,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;oBACtC,MAAM;gBACR,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAExD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,mBAAmB,SAAS,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1E,UAAU,GAAG,YAAY,CAAC;gBAC1B,MAAM;YACR,CAAC;YAED,MAAM,YAAY,GAAc,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;oBAAE,SAAS;gBAE9C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAClD,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,QAAQ,EAAE,CAAC;oBACb,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;oBAC3C,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;wBAC9B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC;oBACtC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,YAAY,CAAC,CAAC;YACnF,WAAW,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;YAEtC,SAAS,EAAE,CAAC;YAEZ,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,UAAU,SAAS,SAAS,QAAQ,IAAI;gBACnD,YAAY,YAAY,CAAC,MAAM,YAAY,gBAAgB,CAAC,MAAM,IAAI;gBACtE,aAAa,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CACjD,CAAC;YAEF,IAAI,MAAM,CAAC,qBAAqB,IAAI,kBAAkB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CACzC,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,EACd,MAAM,CAAC,SAAS,EAChB,gBAAgB,CAAC,MAAM,CACxB,CAAC;gBACF,MAAM,kBAAkB,CAAC,aAAa,CAAC,CAAC;gBACxC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,6CAA6C,SAAS,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7D,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,4BAA4B,SAAS,QAAQ,CAAC,CAAC;gBAC3E,UAAU,GAAG,UAAU,CAAC;gBACxB,MAAM;YACR,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC1C,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,gBAAgB,CACvF,CAAC;gBACF,UAAU,GAAG,cAAc,CAAC;gBAC5B,MAAM;YACR,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACtB,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,8BAA8B,SAAS,eAAe,CAAC,CAAC;gBACpF,UAAU,GAAG,eAAe,CAAC;gBAC7B,MAAM;YACR,CAAC;YAED,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChD,UAAU,GAAG,WAAW,CAAC;QAC3B,CAAC;QAED,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,yBAAyB,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3E,CAAC;QAED,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,0BAA0B,WAAW,CAAC,MAAM,eAAe,SAAS,QAAQ,CAC1F,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,kBAAkB,GACtB,eAAe;YACf,CAAC,CAAC,eAAe,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC;QACpF,MAAM,qBAAqB,GACzB,UAAU,KAAK,cAAc;YAC3B,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC;YACjE,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAC3C,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,EACtB,WAAW,EACX,WAAW,CAAC,MAAM,CACnB,CAAC;QACF,eAAe,CAAC,gBAAgB,GAAG,kBAAkB,CAAC;QAEtD,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,UAAU,EAAE,eAAe;YAC3B,SAAS,EAAE,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACrD,qBAAqB;SACtB,CAAC;IACJ,CAAC;CACF"}
|