@lobu/connector-sdk 6.1.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/acquire.d.ts +7 -0
- package/dist/browser/acquire.d.ts.map +1 -1
- package/dist/browser/acquire.js +68 -16
- package/dist/browser/acquire.js.map +1 -1
- package/dist/browser/cdp-page.d.ts.map +1 -1
- package/dist/browser/cdp-page.js +111 -29
- package/dist/browser/cdp-page.js.map +1 -1
- package/dist/browser/cdp.d.ts.map +1 -1
- package/dist/browser/cdp.js +10 -7
- package/dist/browser/cdp.js.map +1 -1
- package/dist/browser/devtools-active-port.d.ts +20 -0
- package/dist/browser/devtools-active-port.d.ts.map +1 -0
- package/dist/browser/devtools-active-port.js +46 -0
- package/dist/browser/devtools-active-port.js.map +1 -0
- package/dist/browser/launcher.d.ts +1 -2
- package/dist/browser/launcher.d.ts.map +1 -1
- package/dist/browser/launcher.js +12 -9
- package/dist/browser/launcher.js.map +1 -1
- package/dist/browser/mirror-cookies.d.ts +64 -0
- package/dist/browser/mirror-cookies.d.ts.map +1 -0
- package/dist/browser/mirror-cookies.js +293 -0
- package/dist/browser/mirror-cookies.js.map +1 -0
- package/dist/browser-network.d.ts +6 -0
- package/dist/browser-network.d.ts.map +1 -1
- package/dist/browser-network.js +106 -35
- package/dist/browser-network.js.map +1 -1
- package/dist/connector-types.d.ts +32 -0
- package/dist/connector-types.d.ts.map +1 -1
- package/dist/connector-types.js.map +1 -1
- package/dist/index.d.ts +2 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -6
- package/dist/index.js.map +1 -1
- package/dist/retry.d.ts.map +1 -1
- package/dist/retry.js +59 -86
- package/dist/retry.js.map +1 -1
- package/dist/types.d.ts +0 -134
- package/dist/types.d.ts.map +1 -1
- package/package.json +19 -3
- package/dist/api-paginated.d.ts +0 -79
- package/dist/api-paginated.d.ts.map +0 -1
- package/dist/api-paginated.js +0 -120
- package/dist/api-paginated.js.map +0 -1
- package/dist/base.d.ts +0 -65
- package/dist/base.d.ts.map +0 -1
- package/dist/base.js +0 -122
- package/dist/base.js.map +0 -1
- package/dist/browser-paginated.d.ts +0 -141
- package/dist/browser-paginated.d.ts.map +0 -1
- package/dist/browser-paginated.js +0 -269
- package/dist/browser-paginated.js.map +0 -1
- package/dist/http.d.ts +0 -18
- package/dist/http.d.ts.map +0 -1
- package/dist/http.js +0 -74
- package/dist/http.js.map +0 -1
- package/dist/paginated.d.ts +0 -93
- package/dist/paginated.d.ts.map +0 -1
- package/dist/paginated.js +0 -167
- package/dist/paginated.js.map +0 -1
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { TObject } from '@sinclair/typebox';
|
|
2
1
|
/**
|
|
3
2
|
* Checkpoint data structure for tracking feed sync state
|
|
4
3
|
*/
|
|
@@ -30,11 +29,6 @@ export interface FeedSyncResult {
|
|
|
30
29
|
*/
|
|
31
30
|
auth_update?: Record<string, any>;
|
|
32
31
|
}
|
|
33
|
-
export interface ParentFeedDefinition {
|
|
34
|
-
type: string;
|
|
35
|
-
options: FeedOptions;
|
|
36
|
-
description?: string;
|
|
37
|
-
}
|
|
38
32
|
/**
|
|
39
33
|
* Extracted content from platform
|
|
40
34
|
*/
|
|
@@ -51,15 +45,6 @@ export interface Content {
|
|
|
51
45
|
origin_parent_id?: string | null;
|
|
52
46
|
metadata?: Record<string, any>;
|
|
53
47
|
}
|
|
54
|
-
/**
|
|
55
|
-
* Search result from platform search
|
|
56
|
-
*/
|
|
57
|
-
export interface SearchResult {
|
|
58
|
-
url: string;
|
|
59
|
-
title: string;
|
|
60
|
-
description: string;
|
|
61
|
-
metadata?: Record<string, any>;
|
|
62
|
-
}
|
|
63
48
|
/**
|
|
64
49
|
* Feed options passed from MCP tool
|
|
65
50
|
*/
|
|
@@ -128,123 +113,4 @@ export interface Env {
|
|
|
128
113
|
* At runtime, DB values override env defaults
|
|
129
114
|
*/
|
|
130
115
|
export type SessionState = Record<string, any>;
|
|
131
|
-
/**
|
|
132
|
-
* Auth field definition for connector environment keys
|
|
133
|
-
*/
|
|
134
|
-
export interface FeedAuthEnvField {
|
|
135
|
-
key: string;
|
|
136
|
-
label?: string;
|
|
137
|
-
description?: string;
|
|
138
|
-
example?: string;
|
|
139
|
-
secret?: boolean;
|
|
140
|
-
}
|
|
141
|
-
export interface FeedAuthNoneMethod {
|
|
142
|
-
type: 'none';
|
|
143
|
-
}
|
|
144
|
-
export interface FeedAuthEnvKeysMethod {
|
|
145
|
-
type: 'env_keys';
|
|
146
|
-
required?: boolean;
|
|
147
|
-
scope?: 'connection' | 'organization';
|
|
148
|
-
fields: FeedAuthEnvField[];
|
|
149
|
-
description?: string;
|
|
150
|
-
}
|
|
151
|
-
export interface FeedAuthOAuthMethod {
|
|
152
|
-
type: 'oauth';
|
|
153
|
-
provider: string;
|
|
154
|
-
requiredScopes: string[];
|
|
155
|
-
optionalScopes?: string[];
|
|
156
|
-
required?: boolean;
|
|
157
|
-
scope?: 'connection' | 'organization';
|
|
158
|
-
description?: string;
|
|
159
|
-
authorizationUrl?: string;
|
|
160
|
-
tokenUrl?: string;
|
|
161
|
-
userinfoUrl?: string;
|
|
162
|
-
authParams?: Record<string, string>;
|
|
163
|
-
tokenEndpointAuthMethod?: 'client_secret_post' | 'client_secret_basic' | 'none';
|
|
164
|
-
usePkce?: boolean;
|
|
165
|
-
loginScopes?: string[];
|
|
166
|
-
clientIdKey?: string;
|
|
167
|
-
clientSecretKey?: string;
|
|
168
|
-
setupInstructions?: string;
|
|
169
|
-
loginProvisioning?: {
|
|
170
|
-
autoCreateConnection?: boolean;
|
|
171
|
-
};
|
|
172
|
-
}
|
|
173
|
-
export interface FeedAuthBrowserMethod {
|
|
174
|
-
type: 'browser';
|
|
175
|
-
required?: boolean;
|
|
176
|
-
description?: string;
|
|
177
|
-
capture?: 'cli';
|
|
178
|
-
}
|
|
179
|
-
export type FeedAuthMethod = FeedAuthNoneMethod | FeedAuthEnvKeysMethod | FeedAuthOAuthMethod | FeedAuthBrowserMethod;
|
|
180
|
-
export interface FeedAuthSchema {
|
|
181
|
-
methods: FeedAuthMethod[];
|
|
182
|
-
}
|
|
183
|
-
/**
|
|
184
|
-
* Main feed interface
|
|
185
|
-
*/
|
|
186
|
-
export interface IFeed {
|
|
187
|
-
/**
|
|
188
|
-
* Unique identifier for this feed type
|
|
189
|
-
*/
|
|
190
|
-
readonly type: string;
|
|
191
|
-
/**
|
|
192
|
-
* Human-readable display name for this feed
|
|
193
|
-
*/
|
|
194
|
-
readonly displayName: string;
|
|
195
|
-
/**
|
|
196
|
-
* API type: 'api' for HTTP/REST APIs, 'browser' for browser rendering
|
|
197
|
-
*/
|
|
198
|
-
readonly apiType: 'api' | 'browser';
|
|
199
|
-
/**
|
|
200
|
-
* Feed mode: 'entity' for platforms with specific pages (repos, subreddits, companies)
|
|
201
|
-
* or 'search' for query-based platforms (Hacker News, Twitter search)
|
|
202
|
-
*/
|
|
203
|
-
readonly feedMode: 'entity' | 'search';
|
|
204
|
-
/**
|
|
205
|
-
* TypeBox schema for validating feed options
|
|
206
|
-
*/
|
|
207
|
-
readonly optionsSchema: TObject;
|
|
208
|
-
/**
|
|
209
|
-
* Default SQL formula to calculate normalized score (0-100)
|
|
210
|
-
* Can reference: f.score, f.content_length, f.metadata, f.occurred_at
|
|
211
|
-
* Can use window functions like PERCENT_RANK()
|
|
212
|
-
* User can override this per-connection via connections.scoring_formula
|
|
213
|
-
*/
|
|
214
|
-
readonly defaultScoringFormula: string;
|
|
215
|
-
/**
|
|
216
|
-
* Pull new content from platform
|
|
217
|
-
*/
|
|
218
|
-
pull(options: FeedOptions, checkpoint: Checkpoint | null, env: Env, sessionState?: SessionState | null, updateCheckpointFn?: (checkpoint: Checkpoint) => Promise<void>): Promise<FeedSyncResult>;
|
|
219
|
-
/**
|
|
220
|
-
* Validate feed options before saving to database
|
|
221
|
-
*/
|
|
222
|
-
validateOptions(options: FeedOptions): string | null;
|
|
223
|
-
/**
|
|
224
|
-
* Get rate limit information for this platform
|
|
225
|
-
*/
|
|
226
|
-
getRateLimit(): {
|
|
227
|
-
requests_per_minute: number;
|
|
228
|
-
requests_per_hour?: number;
|
|
229
|
-
recommended_interval_ms: number;
|
|
230
|
-
};
|
|
231
|
-
/**
|
|
232
|
-
* Search platform for entities
|
|
233
|
-
* Optional method - not all platforms may support search
|
|
234
|
-
*/
|
|
235
|
-
search?(searchTerm: string, env: Env): Promise<SearchResult[]>;
|
|
236
|
-
/**
|
|
237
|
-
* Generate a URL for the connection from options
|
|
238
|
-
*/
|
|
239
|
-
urlFromOptions(options: FeedOptions): string;
|
|
240
|
-
/**
|
|
241
|
-
* Generate a human-readable display label from options
|
|
242
|
-
*/
|
|
243
|
-
displayLabelFromOptions(options: FeedOptions): string;
|
|
244
|
-
/**
|
|
245
|
-
* Return parent feed definitions required to preserve hierarchy.
|
|
246
|
-
*/
|
|
247
|
-
getParentFeedDefinitions(options: FeedOptions): ParentFeedDefinition[];
|
|
248
|
-
readonly authSchema?: FeedAuthSchema;
|
|
249
|
-
}
|
|
250
116
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,UAAU;IAEzB,cAAc,CAAC,EAAE,IAAI,CAAC;IAGtB,UAAU,EAAE,IAAI,CAAC;IACjB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAGhC;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,UAAU,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,EAAE;QACT,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,wBAAwB,CAAC,EAAE,IAAI,CAAC;QAChC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;KACpB,CAAC;IACF;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,IAAI,CAAC;IAGlB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,KAAK,EAAE,MAAM,CAAC;IAGd,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAKjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,GAAG;IAElB,WAAW,EAAE,MAAM,CAAC;IACpB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAK1B,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,+BAA+B,CAAC,EAAE,MAAM,CAAC;IACzC,+BAA+B,CAAC,EAAE,MAAM,CAAC;IAGzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAGhC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lobu/connector-sdk",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.1.0",
|
|
4
4
|
"description": "Lobu memory SDK — build pluggable connectors for Lobu memory",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,6 +15,18 @@
|
|
|
15
15
|
"types": "./dist/index.d.ts",
|
|
16
16
|
"default": "./dist/index.js"
|
|
17
17
|
}
|
|
18
|
+
},
|
|
19
|
+
"./browser-mirror": {
|
|
20
|
+
"import": {
|
|
21
|
+
"types": "./dist/browser/mirror-cookies.d.ts",
|
|
22
|
+
"default": "./dist/browser/mirror-cookies.js"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"./browser-devtools-active-port": {
|
|
26
|
+
"import": {
|
|
27
|
+
"types": "./dist/browser/devtools-active-port.d.ts",
|
|
28
|
+
"default": "./dist/browser/devtools-active-port.js"
|
|
29
|
+
}
|
|
18
30
|
}
|
|
19
31
|
},
|
|
20
32
|
"files": [
|
|
@@ -26,16 +38,20 @@
|
|
|
26
38
|
"clean": "rm -rf dist"
|
|
27
39
|
},
|
|
28
40
|
"dependencies": {
|
|
29
|
-
"@lobu/core": "
|
|
41
|
+
"@lobu/core": "7.1.0",
|
|
30
42
|
"@sinclair/typebox": "^0.34.41",
|
|
31
43
|
"ky": "^1.14.0"
|
|
32
44
|
},
|
|
33
45
|
"peerDependencies": {
|
|
34
|
-
"playwright": "npm:patchright@^1.57.0"
|
|
46
|
+
"playwright": "npm:patchright@^1.57.0",
|
|
47
|
+
"playwright-vanilla": "npm:playwright@^1.60.0"
|
|
35
48
|
},
|
|
36
49
|
"peerDependenciesMeta": {
|
|
37
50
|
"playwright": {
|
|
38
51
|
"optional": true
|
|
52
|
+
},
|
|
53
|
+
"playwright-vanilla": {
|
|
54
|
+
"optional": true
|
|
39
55
|
}
|
|
40
56
|
},
|
|
41
57
|
"devDependencies": {
|
package/dist/api-paginated.d.ts
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* API Paginated Feed Base Class
|
|
3
|
-
*
|
|
4
|
-
* Extends PaginatedFeed for HTTP/REST API-based feeds.
|
|
5
|
-
* Provides HTTP client setup and common API patterns.
|
|
6
|
-
*/
|
|
7
|
-
import type { KyInstance } from 'ky';
|
|
8
|
-
import { HTTPError } from 'ky';
|
|
9
|
-
import type { PageFetchResult, PaginatedCheckpoint } from './paginated.js';
|
|
10
|
-
import { PaginatedFeed } from './paginated.js';
|
|
11
|
-
import type { Env, FeedOptions, SessionState } from './types.js';
|
|
12
|
-
/**
|
|
13
|
-
* API session state for OAuth/token-based feeds
|
|
14
|
-
*/
|
|
15
|
-
export interface ApiSessionState extends SessionState {
|
|
16
|
-
/** OAuth/API access token */
|
|
17
|
-
access_token?: string;
|
|
18
|
-
/** OAuth refresh token (for token refresh flows) */
|
|
19
|
-
refresh_token?: string;
|
|
20
|
-
/** Token type (e.g., 'Bearer') */
|
|
21
|
-
token_type?: string;
|
|
22
|
-
/** Token expiration time (ISO string) */
|
|
23
|
-
expires_at?: string;
|
|
24
|
-
/** Additional headers to include in requests */
|
|
25
|
-
headers?: Record<string, string>;
|
|
26
|
-
/** API key (alternative to OAuth tokens) */
|
|
27
|
-
api_key?: string;
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Base class for API-based feeds with HTTP pagination
|
|
31
|
-
*/
|
|
32
|
-
export declare abstract class ApiPaginatedFeed<TItem, TResponse = unknown, TCheckpoint extends PaginatedCheckpoint = PaginatedCheckpoint> extends PaginatedFeed<TItem, TCheckpoint> {
|
|
33
|
-
readonly apiType: "api";
|
|
34
|
-
/**
|
|
35
|
-
* Session state for this sync session
|
|
36
|
-
*/
|
|
37
|
-
protected _sessionState: ApiSessionState | null;
|
|
38
|
-
/**
|
|
39
|
-
* Set session state for this sync session
|
|
40
|
-
*/
|
|
41
|
-
protected setSessionState(sessionState: SessionState | null | undefined): void;
|
|
42
|
-
/**
|
|
43
|
-
* Get current session state
|
|
44
|
-
*/
|
|
45
|
-
protected getSessionState(): ApiSessionState | null;
|
|
46
|
-
/**
|
|
47
|
-
* ABSTRACT: Build URL for fetching a specific page
|
|
48
|
-
*/
|
|
49
|
-
protected abstract buildPageUrl(cursor: string | null, options: FeedOptions): string;
|
|
50
|
-
/**
|
|
51
|
-
* ABSTRACT: Parse API response into items and next token
|
|
52
|
-
*/
|
|
53
|
-
protected abstract parseResponse(response: TResponse, options: FeedOptions): PageFetchResult<TItem>;
|
|
54
|
-
/**
|
|
55
|
-
* Get configured HTTP client for this feed
|
|
56
|
-
*/
|
|
57
|
-
protected getHttpClient(_env: Env): KyInstance;
|
|
58
|
-
/**
|
|
59
|
-
* Create HTTP client with Bearer token authentication
|
|
60
|
-
*/
|
|
61
|
-
protected createBearerClient(token: string, additionalHeaders?: Record<string, string>): KyInstance;
|
|
62
|
-
/**
|
|
63
|
-
* Create HTTP client with custom headers
|
|
64
|
-
*/
|
|
65
|
-
protected createClientWithHeaders(headers: Record<string, string>): KyInstance;
|
|
66
|
-
/**
|
|
67
|
-
* Create HTTP client from session state
|
|
68
|
-
*/
|
|
69
|
-
protected createClientFromSessionState(additionalHeaders?: Record<string, string>): KyInstance;
|
|
70
|
-
/**
|
|
71
|
-
* Handle HTTP errors with platform-specific messages
|
|
72
|
-
*/
|
|
73
|
-
protected handleHttpError(error: HTTPError, url: string): never;
|
|
74
|
-
/**
|
|
75
|
-
* Default fetchPage implementation using HTTP client with retry
|
|
76
|
-
*/
|
|
77
|
-
protected fetchPage(cursor: string | null, options: FeedOptions, env: Env): Promise<PageFetchResult<TItem>>;
|
|
78
|
-
}
|
|
79
|
-
//# sourceMappingURL=api-paginated.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"api-paginated.d.ts","sourceRoot":"","sources":["../src/api-paginated.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAI/B,OAAO,KAAK,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kCAAkC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,8BAAsB,gBAAgB,CACpC,KAAK,EACL,SAAS,GAAG,OAAO,EACnB,WAAW,SAAS,mBAAmB,GAAG,mBAAmB,CAC7D,SAAQ,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAG,KAAK,CAAU;IAElC;;OAEG;IACH,SAAS,CAAC,aAAa,EAAE,eAAe,GAAG,IAAI,CAAQ;IAEvD;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IAU9E;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,eAAe,GAAG,IAAI;IAInD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,WAAW,GAAG,MAAM;IAEpF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,QAAQ,EAAE,SAAS,EACnB,OAAO,EAAE,WAAW,GACnB,eAAe,CAAC,KAAK,CAAC;IAEzB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,GAAG,GAAG,UAAU;IAI9C;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAC1B,KAAK,EAAE,MAAM,EACb,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACzC,UAAU;IAIb;;OAEG;IACH,SAAS,CAAC,uBAAuB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,UAAU;IAI9E;;OAEG;IACH,SAAS,CAAC,4BAA4B,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,UAAU;IAsB9F;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,KAAK;IAI/D;;OAEG;cACa,SAAS,CACvB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,GAAG,GACP,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;CAsCnC"}
|
package/dist/api-paginated.js
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* API Paginated Feed Base Class
|
|
3
|
-
*
|
|
4
|
-
* Extends PaginatedFeed for HTTP/REST API-based feeds.
|
|
5
|
-
* Provides HTTP client setup and common API patterns.
|
|
6
|
-
*/
|
|
7
|
-
import { HTTPError } from 'ky';
|
|
8
|
-
import { RateLimitError } from './base.js';
|
|
9
|
-
import { createAuthenticatedClient, createHttpClient, httpClient } from './http.js';
|
|
10
|
-
import { sdkLogger } from './logger.js';
|
|
11
|
-
import { PaginatedFeed } from './paginated.js';
|
|
12
|
-
import { withHttpRetry } from './retry.js';
|
|
13
|
-
/**
|
|
14
|
-
* Base class for API-based feeds with HTTP pagination
|
|
15
|
-
*/
|
|
16
|
-
export class ApiPaginatedFeed extends PaginatedFeed {
|
|
17
|
-
apiType = 'api';
|
|
18
|
-
/**
|
|
19
|
-
* Session state for this sync session
|
|
20
|
-
*/
|
|
21
|
-
_sessionState = null;
|
|
22
|
-
/**
|
|
23
|
-
* Set session state for this sync session
|
|
24
|
-
*/
|
|
25
|
-
setSessionState(sessionState) {
|
|
26
|
-
this._sessionState = sessionState || null;
|
|
27
|
-
if (this._sessionState) {
|
|
28
|
-
sdkLogger.debug({ hasToken: !!this._sessionState.access_token, hasApiKey: !!this._sessionState.api_key }, `[${this.type}] Session state set`);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Get current session state
|
|
33
|
-
*/
|
|
34
|
-
getSessionState() {
|
|
35
|
-
return this._sessionState;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Get configured HTTP client for this feed
|
|
39
|
-
*/
|
|
40
|
-
getHttpClient(_env) {
|
|
41
|
-
return httpClient;
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Create HTTP client with Bearer token authentication
|
|
45
|
-
*/
|
|
46
|
-
createBearerClient(token, additionalHeaders) {
|
|
47
|
-
return createAuthenticatedClient(`Bearer ${token}`, additionalHeaders);
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Create HTTP client with custom headers
|
|
51
|
-
*/
|
|
52
|
-
createClientWithHeaders(headers) {
|
|
53
|
-
return createHttpClient({ headers });
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Create HTTP client from session state
|
|
57
|
-
*/
|
|
58
|
-
createClientFromSessionState(additionalHeaders) {
|
|
59
|
-
if (!this._sessionState) {
|
|
60
|
-
return additionalHeaders ? createHttpClient({ headers: additionalHeaders }) : httpClient;
|
|
61
|
-
}
|
|
62
|
-
const headers = {
|
|
63
|
-
...this._sessionState.headers,
|
|
64
|
-
...additionalHeaders,
|
|
65
|
-
};
|
|
66
|
-
if (this._sessionState.access_token) {
|
|
67
|
-
const tokenType = this._sessionState.token_type || 'Bearer';
|
|
68
|
-
return createAuthenticatedClient(`${tokenType} ${this._sessionState.access_token}`, headers);
|
|
69
|
-
}
|
|
70
|
-
if (this._sessionState.api_key) {
|
|
71
|
-
return createAuthenticatedClient(this._sessionState.api_key, headers);
|
|
72
|
-
}
|
|
73
|
-
return Object.keys(headers).length > 0 ? createHttpClient({ headers }) : httpClient;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Handle HTTP errors with platform-specific messages
|
|
77
|
-
*/
|
|
78
|
-
handleHttpError(error, url) {
|
|
79
|
-
this.handleHTTPError(error.response.status, url);
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* Default fetchPage implementation using HTTP client with retry
|
|
83
|
-
*/
|
|
84
|
-
async fetchPage(cursor, options, env) {
|
|
85
|
-
const client = this.getHttpClient(env);
|
|
86
|
-
const url = this.buildPageUrl(cursor, options);
|
|
87
|
-
try {
|
|
88
|
-
const response = await withHttpRetry(async () => client.get(url).json(), {
|
|
89
|
-
operation: `${this.type} API fetch`,
|
|
90
|
-
context: { url, cursor },
|
|
91
|
-
});
|
|
92
|
-
return this.parseResponse(response, options);
|
|
93
|
-
}
|
|
94
|
-
catch (error) {
|
|
95
|
-
if (error instanceof HTTPError) {
|
|
96
|
-
if (error.response.status === 429) {
|
|
97
|
-
const retryAfter = error.response.headers.get('retry-after');
|
|
98
|
-
let retryAfterMs;
|
|
99
|
-
if (retryAfter) {
|
|
100
|
-
const numericRetry = Number(retryAfter);
|
|
101
|
-
if (!Number.isNaN(numericRetry)) {
|
|
102
|
-
retryAfterMs = numericRetry * 1000;
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
const retryDate = Date.parse(retryAfter);
|
|
106
|
-
if (!Number.isNaN(retryDate)) {
|
|
107
|
-
retryAfterMs = retryDate - Date.now();
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
throw new RateLimitError(`${this.displayName} rate limit exceeded. Please wait before retrying.`, retryAfterMs && retryAfterMs > 0 ? retryAfterMs : undefined);
|
|
112
|
-
}
|
|
113
|
-
this.handleHttpError(error, url);
|
|
114
|
-
}
|
|
115
|
-
sdkLogger.error({ error, url }, `[${this.type}] API fetch failed`);
|
|
116
|
-
throw error;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
//# sourceMappingURL=api-paginated.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"api-paginated.js","sourceRoot":"","sources":["../src/api-paginated.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAqB3C;;GAEG;AACH,MAAM,OAAgB,gBAIpB,SAAQ,aAAiC;IAChC,OAAO,GAAG,KAAc,CAAC;IAElC;;OAEG;IACO,aAAa,GAA2B,IAAI,CAAC;IAEvD;;OAEG;IACO,eAAe,CAAC,YAA6C;QACrE,IAAI,CAAC,aAAa,GAAI,YAAgC,IAAI,IAAI,CAAC;QAC/D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,SAAS,CAAC,KAAK,CACb,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EACxF,IAAI,IAAI,CAAC,IAAI,qBAAqB,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACO,eAAe;QACvB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAeD;;OAEG;IACO,aAAa,CAAC,IAAS;QAC/B,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACO,kBAAkB,CAC1B,KAAa,EACb,iBAA0C;QAE1C,OAAO,yBAAyB,CAAC,UAAU,KAAK,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACO,uBAAuB,CAAC,OAA+B;QAC/D,OAAO,gBAAgB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,4BAA4B,CAAC,iBAA0C;QAC/E,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,OAAO,iBAAiB,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAC3F,CAAC;QAED,MAAM,OAAO,GAA2B;YACtC,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO;YAC7B,GAAG,iBAAiB;SACrB,CAAC;QAEF,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,QAAQ,CAAC;YAC5D,OAAO,yBAAyB,CAAC,GAAG,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;YAC/B,OAAO,yBAAyB,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IACtF,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,KAAgB,EAAE,GAAW;QACrD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,SAAS,CACvB,MAAqB,EACrB,OAAoB,EACpB,GAAQ;QAER,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE/C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAa,EAAE;gBAClF,SAAS,EAAE,GAAG,IAAI,CAAC,IAAI,YAAY;gBACnC,OAAO,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE;aACzB,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;gBAC/B,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAClC,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBAC7D,IAAI,YAAgC,CAAC;oBACrC,IAAI,UAAU,EAAE,CAAC;wBACf,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;wBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;4BAChC,YAAY,GAAG,YAAY,GAAG,IAAI,CAAC;wBACrC,CAAC;6BAAM,CAAC;4BACN,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;gCAC7B,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;4BACxC,CAAC;wBACH,CAAC;oBACH,CAAC;oBACD,MAAM,IAAI,cAAc,CACtB,GAAG,IAAI,CAAC,WAAW,oDAAoD,EACvE,YAAY,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAC5D,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,oBAAoB,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
package/dist/base.d.ts
DELETED
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import type { TObject } from '@sinclair/typebox';
|
|
2
|
-
import type { Checkpoint, Content, Env, FeedAuthSchema, FeedOptions, FeedSyncResult, IFeed, ParentFeedDefinition, SessionState } from './types.js';
|
|
3
|
-
export declare class RateLimitError extends Error {
|
|
4
|
-
readonly retryAfterMs?: number;
|
|
5
|
-
constructor(message: string, retryAfterMs?: number);
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Base feed implementation with common functionality
|
|
9
|
-
* All platform-specific feeds should extend this class
|
|
10
|
-
*/
|
|
11
|
-
export declare abstract class BaseFeed implements IFeed {
|
|
12
|
-
abstract readonly type: string;
|
|
13
|
-
abstract readonly displayName: string;
|
|
14
|
-
abstract readonly apiType: 'api' | 'browser';
|
|
15
|
-
abstract readonly feedMode: 'entity' | 'search';
|
|
16
|
-
abstract readonly optionsSchema: TObject;
|
|
17
|
-
abstract readonly defaultScoringFormula: string;
|
|
18
|
-
readonly authSchema: FeedAuthSchema;
|
|
19
|
-
abstract pull(options: FeedOptions, checkpoint: Checkpoint | null, env: Env, sessionState?: SessionState | null, updateCheckpointFn?: (checkpoint: Checkpoint) => Promise<void>): Promise<FeedSyncResult>;
|
|
20
|
-
abstract urlFromOptions(options: FeedOptions): string;
|
|
21
|
-
abstract displayLabelFromOptions(options: FeedOptions): string;
|
|
22
|
-
abstract validateOptions(options: FeedOptions): string | null;
|
|
23
|
-
getParentFeedDefinitions(_options: FeedOptions): ParentFeedDefinition[];
|
|
24
|
-
/**
|
|
25
|
-
* Validate options using TypeBox schema
|
|
26
|
-
* Subclasses can call this for schema validation before adding custom business logic
|
|
27
|
-
*/
|
|
28
|
-
protected validateWithSchema(options: FeedOptions): string | null;
|
|
29
|
-
/**
|
|
30
|
-
* Get rate limit information for this platform
|
|
31
|
-
* Override this method in platform-specific feeds
|
|
32
|
-
* Default is conservative: 10 requests per minute
|
|
33
|
-
*/
|
|
34
|
-
getRateLimit(): {
|
|
35
|
-
requests_per_minute: number;
|
|
36
|
-
recommended_interval_ms: number;
|
|
37
|
-
};
|
|
38
|
-
/**
|
|
39
|
-
* Helper to check if content is newer than checkpoint
|
|
40
|
-
*/
|
|
41
|
-
protected isNewerThan(contentDate: Date, checkpoint: Checkpoint | null): boolean;
|
|
42
|
-
/**
|
|
43
|
-
* Calculate lookback date from options
|
|
44
|
-
* @param options - Feed options with optional lookback_days
|
|
45
|
-
* @param defaultDays - Default lookback period (default: 365)
|
|
46
|
-
*/
|
|
47
|
-
protected getLookbackDate(options: FeedOptions, defaultDays?: number): Date;
|
|
48
|
-
/**
|
|
49
|
-
* Sleep for specified milliseconds (for rate limiting)
|
|
50
|
-
*/
|
|
51
|
-
protected sleep(ms: number): Promise<void>;
|
|
52
|
-
/**
|
|
53
|
-
* Check if feed is in incremental mode
|
|
54
|
-
*/
|
|
55
|
-
protected isIncrementalMode(checkpoint: Checkpoint | null, paginationToken?: string | null): boolean;
|
|
56
|
-
/**
|
|
57
|
-
* Helper to deduplicate content by origin_id
|
|
58
|
-
*/
|
|
59
|
-
protected deduplicate(contents: Content[], seenIds: Set<string>): Content[];
|
|
60
|
-
/**
|
|
61
|
-
* Handle HTTP errors with structured logging and platform-specific messages
|
|
62
|
-
*/
|
|
63
|
-
protected handleHTTPError(status: number, context: string, platformName?: string): never;
|
|
64
|
-
}
|
|
65
|
-
//# sourceMappingURL=base.d.ts.map
|
package/dist/base.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../src/base.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,KAAK,EACV,UAAU,EACV,OAAO,EACP,GAAG,EACH,cAAc,EACd,WAAW,EACX,cAAc,EACd,KAAK,EACL,oBAAoB,EACpB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;gBAEnB,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM;CAKnD;AAED;;;GAGG;AACH,8BAAsB,QAAS,YAAW,KAAK;IAC7C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,KAAK,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAChD,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAEhD,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAmC;IAEtE,QAAQ,CAAC,IAAI,CACX,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,GAAG,EAAE,GAAG,EACR,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,EAClC,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAC7D,OAAO,CAAC,cAAc,CAAC;IAE1B,QAAQ,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM;IAErD,QAAQ,CAAC,uBAAuB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM;IAE9D,QAAQ,CAAC,eAAe,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAE7D,wBAAwB,CAAC,QAAQ,EAAE,WAAW,GAAG,oBAAoB,EAAE;IAIvE;;;OAGG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAgBjE;;;;OAIG;IACH,YAAY;;;;IAOZ;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI,GAAG,OAAO;IAKhF;;;;OAIG;IACH,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,GAAE,MAAY,GAAG,IAAI;IAKhF;;OAEG;IACH,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C;;OAEG;IACH,SAAS,CAAC,iBAAiB,CACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAC9B,OAAO;IAIV;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,EAAE;IAS3E;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK;CA+BzF"}
|
package/dist/base.js
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
import { Value } from '@sinclair/typebox/value';
|
|
2
|
-
import { sdkLogger } from './logger.js';
|
|
3
|
-
export class RateLimitError extends Error {
|
|
4
|
-
retryAfterMs;
|
|
5
|
-
constructor(message, retryAfterMs) {
|
|
6
|
-
super(message);
|
|
7
|
-
this.name = 'RateLimitError';
|
|
8
|
-
this.retryAfterMs = retryAfterMs;
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* Base feed implementation with common functionality
|
|
13
|
-
* All platform-specific feeds should extend this class
|
|
14
|
-
*/
|
|
15
|
-
export class BaseFeed {
|
|
16
|
-
authSchema = { methods: [{ type: 'none' }] };
|
|
17
|
-
getParentFeedDefinitions(_options) {
|
|
18
|
-
return [];
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Validate options using TypeBox schema
|
|
22
|
-
* Subclasses can call this for schema validation before adding custom business logic
|
|
23
|
-
*/
|
|
24
|
-
validateWithSchema(options) {
|
|
25
|
-
try {
|
|
26
|
-
const errors = [...Value.Errors(this.optionsSchema, options)];
|
|
27
|
-
if (errors.length > 0) {
|
|
28
|
-
// Format first error for user-friendly message
|
|
29
|
-
const firstError = errors[0];
|
|
30
|
-
const field = firstError.path.replace(/^\//, '');
|
|
31
|
-
return `Invalid option ${field ? `"${field}"` : ''}: ${firstError.message}`;
|
|
32
|
-
}
|
|
33
|
-
return null;
|
|
34
|
-
}
|
|
35
|
-
catch (error) {
|
|
36
|
-
sdkLogger.error({ error }, '[BaseFeed] Schema validation error:');
|
|
37
|
-
return 'Invalid feed options format';
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Get rate limit information for this platform
|
|
42
|
-
* Override this method in platform-specific feeds
|
|
43
|
-
* Default is conservative: 10 requests per minute
|
|
44
|
-
*/
|
|
45
|
-
getRateLimit() {
|
|
46
|
-
return {
|
|
47
|
-
requests_per_minute: 10,
|
|
48
|
-
recommended_interval_ms: 6000, // 6 seconds
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Helper to check if content is newer than checkpoint
|
|
53
|
-
*/
|
|
54
|
-
isNewerThan(contentDate, checkpoint) {
|
|
55
|
-
if (!checkpoint || !checkpoint.last_timestamp)
|
|
56
|
-
return true;
|
|
57
|
-
return contentDate > checkpoint.last_timestamp;
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Calculate lookback date from options
|
|
61
|
-
* @param options - Feed options with optional lookback_days
|
|
62
|
-
* @param defaultDays - Default lookback period (default: 365)
|
|
63
|
-
*/
|
|
64
|
-
getLookbackDate(options, defaultDays = 365) {
|
|
65
|
-
const lookbackDays = options.lookback_days || defaultDays;
|
|
66
|
-
return new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000);
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Sleep for specified milliseconds (for rate limiting)
|
|
70
|
-
*/
|
|
71
|
-
sleep(ms) {
|
|
72
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Check if feed is in incremental mode
|
|
76
|
-
*/
|
|
77
|
-
isIncrementalMode(checkpoint, paginationToken) {
|
|
78
|
-
return !!checkpoint?.last_timestamp && !paginationToken;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Helper to deduplicate content by origin_id
|
|
82
|
-
*/
|
|
83
|
-
deduplicate(contents, seenIds) {
|
|
84
|
-
return contents.filter((c) => {
|
|
85
|
-
if (!c.origin_id)
|
|
86
|
-
return false;
|
|
87
|
-
if (seenIds.has(c.origin_id))
|
|
88
|
-
return false;
|
|
89
|
-
seenIds.add(c.origin_id);
|
|
90
|
-
return true;
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* Handle HTTP errors with structured logging and platform-specific messages
|
|
95
|
-
*/
|
|
96
|
-
handleHTTPError(status, context, platformName) {
|
|
97
|
-
const platform = platformName || this.displayName;
|
|
98
|
-
sdkLogger.error({
|
|
99
|
-
status,
|
|
100
|
-
context,
|
|
101
|
-
platform: this.type,
|
|
102
|
-
timestamp: new Date().toISOString(),
|
|
103
|
-
}, `[${platform}Feed] HTTP ${status} error:`);
|
|
104
|
-
const errorMessages = {
|
|
105
|
-
400: `Bad request to ${platform}: ${context}. Check your feed options.`,
|
|
106
|
-
401: `Authentication failed for ${platform}. Check your API credentials.`,
|
|
107
|
-
403: `Access forbidden to ${platform} resource: ${context}. The resource may be private or require authentication.`,
|
|
108
|
-
404: `Resource not found on ${platform}: ${context}. Verify the resource exists.`,
|
|
109
|
-
422: `Invalid request to ${platform}: ${context}. Check your parameters.`,
|
|
110
|
-
429: `${platform} rate limit exceeded. Please wait before retrying.`,
|
|
111
|
-
500: `${platform} server error (${status}). This is temporary, please retry later.`,
|
|
112
|
-
502: `${platform} bad gateway (${status}). This is temporary, please retry later.`,
|
|
113
|
-
503: `${platform} service unavailable (${status}). This is temporary, please retry later.`,
|
|
114
|
-
};
|
|
115
|
-
const message = errorMessages[status] || `${platform} API error: ${status}`;
|
|
116
|
-
if (status === 429) {
|
|
117
|
-
throw new RateLimitError(message);
|
|
118
|
-
}
|
|
119
|
-
throw new Error(message);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
//# sourceMappingURL=base.js.map
|
package/dist/base.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"base.js","sourceRoot":"","sources":["../src/base.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAaxC,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,YAAY,CAAU;IAE/B,YAAY,OAAe,EAAE,YAAqB;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAgB,QAAQ;IAQnB,UAAU,GAAmB,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAgBtE,wBAAwB,CAAC,QAAqB;QAC5C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACO,kBAAkB,CAAC,OAAoB;QAC/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,+CAA+C;gBAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,OAAO,kBAAkB,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9E,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,qCAAqC,CAAC,CAAC;YAClE,OAAO,6BAA6B,CAAC;QACvC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,OAAO;YACL,mBAAmB,EAAE,EAAE;YACvB,uBAAuB,EAAE,IAAI,EAAE,YAAY;SAC5C,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,WAAiB,EAAE,UAA6B;QACpE,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAC3D,OAAO,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACO,eAAe,CAAC,OAAoB,EAAE,cAAsB,GAAG;QACvE,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,IAAI,WAAW,CAAC;QAC1D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,EAAU;QACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACO,iBAAiB,CACzB,UAA6B,EAC7B,eAA+B;QAE/B,OAAO,CAAC,CAAC,UAAU,EAAE,cAAc,IAAI,CAAC,eAAe,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,QAAmB,EAAE,OAAoB;QAC7D,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,IAAI,CAAC,CAAC,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YAC/B,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAc,EAAE,OAAe,EAAE,YAAqB;QAC9E,MAAM,QAAQ,GAAG,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAElD,SAAS,CAAC,KAAK,CACb;YACE,MAAM;YACN,OAAO;YACP,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,EACD,IAAI,QAAQ,cAAc,MAAM,SAAS,CAC1C,CAAC;QAEF,MAAM,aAAa,GAA2B;YAC5C,GAAG,EAAE,kBAAkB,QAAQ,KAAK,OAAO,4BAA4B;YACvE,GAAG,EAAE,6BAA6B,QAAQ,+BAA+B;YACzE,GAAG,EAAE,uBAAuB,QAAQ,cAAc,OAAO,0DAA0D;YACnH,GAAG,EAAE,yBAAyB,QAAQ,KAAK,OAAO,+BAA+B;YACjF,GAAG,EAAE,sBAAsB,QAAQ,KAAK,OAAO,0BAA0B;YACzE,GAAG,EAAE,GAAG,QAAQ,oDAAoD;YACpE,GAAG,EAAE,GAAG,QAAQ,kBAAkB,MAAM,2CAA2C;YACnF,GAAG,EAAE,GAAG,QAAQ,iBAAiB,MAAM,2CAA2C;YAClF,GAAG,EAAE,GAAG,QAAQ,yBAAyB,MAAM,2CAA2C;SAC3F,CAAC;QAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,eAAe,MAAM,EAAE,CAAC;QAC5E,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;CACF"}
|