@lobu/cli 6.1.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/_lib/apply/apply-cmd.d.ts +36 -0
- package/dist/commands/_lib/apply/apply-cmd.d.ts.map +1 -1
- package/dist/commands/_lib/apply/apply-cmd.js +696 -40
- package/dist/commands/_lib/apply/apply-cmd.js.map +1 -1
- package/dist/commands/_lib/apply/client.d.ts +285 -0
- package/dist/commands/_lib/apply/client.d.ts.map +1 -1
- package/dist/commands/_lib/apply/client.js +469 -28
- package/dist/commands/_lib/apply/client.js.map +1 -1
- package/dist/commands/_lib/apply/desired-state.d.ts +187 -3
- package/dist/commands/_lib/apply/desired-state.d.ts.map +1 -1
- package/dist/commands/_lib/apply/desired-state.js +879 -88
- package/dist/commands/_lib/apply/desired-state.js.map +1 -1
- package/dist/commands/_lib/apply/diff.d.ts +72 -3
- package/dist/commands/_lib/apply/diff.d.ts.map +1 -1
- package/dist/commands/_lib/apply/diff.js +473 -84
- package/dist/commands/_lib/apply/diff.js.map +1 -1
- package/dist/commands/_lib/apply/prompt.d.ts +6 -0
- package/dist/commands/_lib/apply/prompt.d.ts.map +1 -1
- package/dist/commands/_lib/apply/prompt.js +16 -0
- package/dist/commands/_lib/apply/prompt.js.map +1 -1
- package/dist/commands/_lib/apply/render.d.ts +9 -0
- package/dist/commands/_lib/apply/render.d.ts.map +1 -1
- package/dist/commands/_lib/apply/render.js +80 -3
- package/dist/commands/_lib/apply/render.js.map +1 -1
- package/dist/commands/_lib/connector-loader.d.ts +3 -0
- package/dist/commands/_lib/connector-loader.d.ts.map +1 -0
- package/dist/commands/_lib/connector-loader.js +129 -0
- package/dist/commands/_lib/connector-loader.js.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts +35 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.js +351 -0
- package/dist/commands/_lib/connector-run-cmd.js.map +1 -0
- package/dist/commands/_lib/export/export-cmd.d.ts +35 -0
- package/dist/commands/_lib/export/export-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/export/export-cmd.js +329 -0
- package/dist/commands/_lib/export/export-cmd.js.map +1 -0
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +11 -14
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/chat.d.ts.map +1 -1
- package/dist/commands/chat.js +28 -7
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/connector.d.ts +3 -0
- package/dist/commands/connector.d.ts.map +1 -0
- package/dist/commands/connector.js +5 -0
- package/dist/commands/connector.js.map +1 -0
- package/dist/commands/dev.d.ts +23 -0
- package/dist/commands/dev.d.ts.map +1 -1
- package/dist/commands/dev.js +273 -8
- package/dist/commands/dev.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +2 -3
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +28 -18
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +29 -1
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/login.d.ts.map +1 -1
- package/dist/commands/login.js +22 -16
- package/dist/commands/login.js.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.d.ts.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.js +15 -144
- package/dist/commands/memory/_lib/browser-auth-cmd.js.map +1 -1
- package/dist/commands/memory/_lib/schema.d.ts +28 -1
- package/dist/commands/memory/_lib/schema.d.ts.map +1 -1
- package/dist/commands/memory/_lib/schema.js +120 -4
- package/dist/commands/memory/_lib/schema.js.map +1 -1
- package/dist/commands/memory/_lib/seed-cmd.d.ts.map +1 -1
- package/dist/commands/memory/_lib/seed-cmd.js +41 -18
- package/dist/commands/memory/_lib/seed-cmd.js.map +1 -1
- package/dist/commands/org.d.ts +4 -0
- package/dist/commands/org.d.ts.map +1 -1
- package/dist/commands/org.js +10 -0
- package/dist/commands/org.js.map +1 -1
- package/dist/commands/token.d.ts +9 -0
- package/dist/commands/token.d.ts.map +1 -1
- package/dist/commands/token.js +54 -3
- package/dist/commands/token.js.map +1 -1
- package/dist/commands/validate.d.ts.map +1 -1
- package/dist/commands/validate.js +4 -13
- package/dist/commands/validate.js.map +1 -1
- package/dist/config/loader.js +2 -2
- package/dist/config/loader.js.map +1 -1
- package/dist/connectors/README.md +2 -3
- package/dist/connectors/apple_health.ts +138 -0
- package/dist/connectors/apple_photos.ts +178 -0
- package/dist/connectors/apple_screen_time.ts +82 -0
- package/dist/connectors/browser/evaluate.ts +120 -0
- package/dist/connectors/browser/fill_form.ts +107 -0
- package/dist/connectors/browser/page_text.ts +108 -0
- package/dist/connectors/browser-scraper-utils.ts +111 -3
- package/dist/connectors/capterra.ts +5 -1
- package/dist/connectors/chrome_tabs.ts +74 -0
- package/dist/connectors/g2.ts +5 -1
- package/dist/connectors/github.ts +16 -38
- package/dist/connectors/glassdoor.ts +5 -1
- package/dist/connectors/google_calendar.ts +28 -6
- package/dist/connectors/google_gmail.ts +6 -3
- package/dist/connectors/google_play.ts +32 -5
- package/dist/connectors/hackernews.ts +37 -2
- package/dist/connectors/index.ts +14 -1
- package/dist/connectors/linkedin.ts +32 -9
- package/dist/connectors/local_directory.ts +91 -0
- package/dist/connectors/reddit.ts +1 -0
- package/dist/connectors/revolut.ts +569 -0
- package/dist/connectors/rss.ts +33 -8
- package/dist/connectors/trustpilot.ts +36 -21
- package/dist/connectors/website.ts +8 -69
- package/dist/connectors/whatsapp.ts +21 -22
- package/dist/connectors/whatsapp_local.ts +125 -0
- package/dist/connectors/x.ts +17 -7
- package/dist/db/migrations/20260510220000_connector_required_capability.sql +47 -0
- package/dist/db/migrations/20260512000000_device_worker_connection_binding.sql +113 -0
- package/dist/db/migrations/20260512131703_connections_slug.sql +131 -0
- package/dist/db/migrations/20260513000000_chat_user_identities.sql +24 -0
- package/dist/db/migrations/20260513120000_auth_profiles_device_binding.sql +50 -0
- package/dist/db/migrations/20260513150000_auth_profiles_cdp_url.sql +43 -0
- package/dist/db/migrations/20260513200000_notifications_as_events.sql +86 -0
- package/dist/db/migrations/20260514000000_scheduled_jobs.sql +97 -0
- package/dist/db/migrations/20260514120000_auth_profiles_connector_key_nullable.sql +42 -0
- package/dist/db/migrations/20260514130000_connection_action_modes.sql +103 -0
- package/dist/db/migrations/20260514160000_auth_profiles_mirror_mode.sql +32 -0
- package/dist/db/migrations/20260515120000_agents_per_org_pk.sql +66 -0
- package/dist/db/migrations/20260515150000_geo_enrichment.sql +208 -0
- package/dist/db/migrations/20260515160000_drop_agents_org_id_unique.sql +24 -0
- package/dist/db/migrations/20260515170000_auth_profiles_default_for_connector.sql +23 -0
- package/dist/db/migrations/20260516120000_agents_per_org_pk_swap.sql +125 -0
- package/dist/db/migrations/20260516200000_events_search_tsv.sql +134 -0
- package/dist/db/migrations/20260516200100_events_lifecycle_changes_index.sql +25 -0
- package/dist/db/migrations/20260517010000_drop_unused_indexes.sql +49 -0
- package/dist/db/migrations/20260517020000_softdelete_orphan_feeds.sql +56 -0
- package/dist/db/migrations/20260517030000_pat_worker_id_binding.sql +27 -0
- package/dist/db/migrations/20260517040000_archive_orphan_watchers.sql +30 -0
- package/dist/db/migrations/20260517050000_watcher_agent_id_not_null.sql +34 -0
- package/dist/db/migrations/20260517060000_watcher_schema_additions.sql +78 -0
- package/dist/db/migrations/20260517150000_goals_primitive.sql +55 -0
- package/dist/db/migrations/20260517160000_drop_goals_primitive.sql +45 -0
- package/dist/db/migrations/20260518000000_pending_interactions.sql +49 -0
- package/dist/db/migrations/20260518010000_runs_heartbeat_reaper_index.sql +22 -0
- package/dist/eval/client.d.ts.map +1 -1
- package/dist/eval/client.js +11 -0
- package/dist/eval/client.js.map +1 -1
- package/dist/eval/grader.js +2 -1
- package/dist/eval/grader.js.map +1 -1
- package/dist/eval/types.d.ts +2 -0
- package/dist/eval/types.d.ts.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +115 -114
- package/dist/index.js.map +1 -1
- package/dist/internal/context.d.ts +9 -0
- package/dist/internal/context.d.ts.map +1 -1
- package/dist/internal/context.js +41 -6
- package/dist/internal/context.js.map +1 -1
- package/dist/internal/credentials.d.ts +5 -0
- package/dist/internal/credentials.d.ts.map +1 -1
- package/dist/internal/credentials.js +75 -1
- package/dist/internal/credentials.js.map +1 -1
- package/dist/internal/gateway-url.d.ts +14 -0
- package/dist/internal/gateway-url.d.ts.map +1 -1
- package/dist/internal/gateway-url.js +19 -0
- package/dist/internal/gateway-url.js.map +1 -1
- package/dist/internal/index.d.ts +1 -1
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +1 -1
- package/dist/internal/index.js.map +1 -1
- package/dist/internal/local-env.d.ts.map +1 -1
- package/dist/internal/local-env.js +9 -2
- package/dist/internal/local-env.js.map +1 -1
- package/dist/server.bundle.mjs +42251 -36931
- package/dist/start-local.bundle.mjs +16437 -9882
- package/dist/templates/TESTING.md.tmpl +9 -9
- package/package.json +8 -6
- package/dist/connectors/google_photos.ts +0 -776
|
@@ -53,6 +53,14 @@ interface RepoRef {
|
|
|
53
53
|
repo: string;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
interface GitHubMutationResponse {
|
|
57
|
+
id: number;
|
|
58
|
+
number: number;
|
|
59
|
+
html_url: string;
|
|
60
|
+
state: string;
|
|
61
|
+
draft?: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
56
64
|
interface GitHubRepositoryLike {
|
|
57
65
|
id?: number;
|
|
58
66
|
full_name?: string;
|
|
@@ -168,13 +176,6 @@ function toIsoOrUndefined(value: unknown): string | undefined {
|
|
|
168
176
|
return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString();
|
|
169
177
|
}
|
|
170
178
|
|
|
171
|
-
function stripMarkdown(code: string): string {
|
|
172
|
-
return code
|
|
173
|
-
.replace(/```[a-zA-Z]*\n?/g, '')
|
|
174
|
-
.replace(/```/g, '')
|
|
175
|
-
.trim();
|
|
176
|
-
}
|
|
177
|
-
|
|
178
179
|
const REPO_PROPS = {
|
|
179
180
|
repo_owner: { type: 'string', minLength: 1, description: 'Repository owner' },
|
|
180
181
|
repo_name: { type: 'string', minLength: 1, description: 'Repository name' },
|
|
@@ -207,6 +208,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
207
208
|
name: 'GitHub',
|
|
208
209
|
description: 'Collects GitHub issues/discussions and executes repo actions.',
|
|
209
210
|
version: '1.2.0',
|
|
211
|
+
faviconDomain: 'github.com',
|
|
210
212
|
authSchema: {
|
|
211
213
|
methods: [
|
|
212
214
|
{
|
|
@@ -569,7 +571,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
569
571
|
};
|
|
570
572
|
|
|
571
573
|
async sync(ctx: SyncContext): Promise<SyncResult> {
|
|
572
|
-
const config =
|
|
574
|
+
const config = ctx.config as GitHubConfig;
|
|
573
575
|
const repo = this.resolveRepo(config, {});
|
|
574
576
|
const token = this.resolveToken(ctx.credentials?.accessToken, config);
|
|
575
577
|
const contentType = (ctx.feedKey ?? 'issues') as GitHubContentType;
|
|
@@ -611,7 +613,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
611
613
|
|
|
612
614
|
async execute(ctx: ActionContext): Promise<ActionResult> {
|
|
613
615
|
try {
|
|
614
|
-
const config =
|
|
616
|
+
const config = ctx.config as GitHubConfig;
|
|
615
617
|
const repo = this.resolveRepo(config, ctx.input);
|
|
616
618
|
const token = this.resolveToken(ctx.credentials?.accessToken, config);
|
|
617
619
|
|
|
@@ -643,10 +645,6 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
643
645
|
}
|
|
644
646
|
}
|
|
645
647
|
|
|
646
|
-
private parseConfig(raw: Record<string, unknown>): GitHubConfig {
|
|
647
|
-
return raw as GitHubConfig;
|
|
648
|
-
}
|
|
649
|
-
|
|
650
648
|
private resolveRepo(config: GitHubConfig, input: Record<string, unknown>): RepoRef {
|
|
651
649
|
const owner = asString(input.repo_owner) ?? config.repo_owner;
|
|
652
650
|
const repo = asString(input.repo_name) ?? config.repo_name;
|
|
@@ -686,7 +684,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
686
684
|
|
|
687
685
|
private async syncContent(params: {
|
|
688
686
|
repo: RepoRef;
|
|
689
|
-
contentType: GitHubContentType
|
|
687
|
+
contentType: Exclude<GitHubContentType, 'stargazers'>;
|
|
690
688
|
sinceIso: string;
|
|
691
689
|
labelsFilter: string[];
|
|
692
690
|
token: string | null;
|
|
@@ -705,10 +703,6 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
705
703
|
return await this.syncDiscussions(repo, sinceIso, token);
|
|
706
704
|
case 'discussion_comments':
|
|
707
705
|
return await this.syncDiscussionComments(repo, sinceIso, token);
|
|
708
|
-
case 'stargazers':
|
|
709
|
-
return [];
|
|
710
|
-
default:
|
|
711
|
-
return [];
|
|
712
706
|
}
|
|
713
707
|
}
|
|
714
708
|
|
|
@@ -1327,12 +1321,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1327
1321
|
? input.assignees.filter((value): value is string => typeof value === 'string')
|
|
1328
1322
|
: undefined;
|
|
1329
1323
|
|
|
1330
|
-
const issue = await this.requestJson<{
|
|
1331
|
-
id: number;
|
|
1332
|
-
number: number;
|
|
1333
|
-
html_url: string;
|
|
1334
|
-
state: string;
|
|
1335
|
-
}>({
|
|
1324
|
+
const issue = await this.requestJson<GitHubMutationResponse>({
|
|
1336
1325
|
method: 'POST',
|
|
1337
1326
|
url: `https://api.github.com/repos/${repo.owner}/${repo.repo}/issues`,
|
|
1338
1327
|
token,
|
|
@@ -1394,12 +1383,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1394
1383
|
const issueNumber = toInt(input.issue_number, 0);
|
|
1395
1384
|
if (!issueNumber) return { success: false, error: 'issue_number is required' };
|
|
1396
1385
|
|
|
1397
|
-
const issue = await this.requestJson<{
|
|
1398
|
-
id: number;
|
|
1399
|
-
number: number;
|
|
1400
|
-
html_url: string;
|
|
1401
|
-
state: string;
|
|
1402
|
-
}>({
|
|
1386
|
+
const issue = await this.requestJson<GitHubMutationResponse>({
|
|
1403
1387
|
method: 'PATCH',
|
|
1404
1388
|
url: `https://api.github.com/repos/${repo.owner}/${repo.repo}/issues/${issueNumber}`,
|
|
1405
1389
|
token,
|
|
@@ -1432,13 +1416,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1432
1416
|
const body = asString(input.body);
|
|
1433
1417
|
const draft = typeof input.draft === 'boolean' ? input.draft : undefined;
|
|
1434
1418
|
|
|
1435
|
-
const pr = await this.requestJson<{
|
|
1436
|
-
id: number;
|
|
1437
|
-
number: number;
|
|
1438
|
-
html_url: string;
|
|
1439
|
-
state: string;
|
|
1440
|
-
draft?: boolean;
|
|
1441
|
-
}>({
|
|
1419
|
+
const pr = await this.requestJson<GitHubMutationResponse>({
|
|
1442
1420
|
method: 'POST',
|
|
1443
1421
|
url: `https://api.github.com/repos/${repo.owner}/${repo.repo}/pulls`,
|
|
1444
1422
|
token,
|
|
@@ -1489,7 +1467,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
1489
1467
|
? mergeMethod
|
|
1490
1468
|
: undefined,
|
|
1491
1469
|
commit_title: commitTitle,
|
|
1492
|
-
commit_message: commitMessage
|
|
1470
|
+
commit_message: commitMessage,
|
|
1493
1471
|
},
|
|
1494
1472
|
});
|
|
1495
1473
|
|
|
@@ -16,6 +16,8 @@ import {
|
|
|
16
16
|
type SyncResult,
|
|
17
17
|
} from '@lobu/connector-sdk';
|
|
18
18
|
import {
|
|
19
|
+
getBrowserCdpUrl,
|
|
20
|
+
getBrowserUserDataDir,
|
|
19
21
|
handleCookieConsent,
|
|
20
22
|
openStealthBrowser,
|
|
21
23
|
validateUrlDomain,
|
|
@@ -158,7 +160,9 @@ export default class GlassdoorConnector extends ConnectorRuntime {
|
|
|
158
160
|
: `https://www.glassdoor.com/Reviews/${company_name}-reviews-SRCH_KE0.htm`;
|
|
159
161
|
validateUrlDomain(baseUrl, 'glassdoor.com');
|
|
160
162
|
|
|
161
|
-
const
|
|
163
|
+
const userDataDir = getBrowserUserDataDir(ctx.sessionState);
|
|
164
|
+
const cdpUrl = getBrowserCdpUrl(ctx.sessionState) ?? 'auto';
|
|
165
|
+
const session = await openStealthBrowser({ cdpUrl, userDataDir });
|
|
162
166
|
|
|
163
167
|
return withBrowserErrorCapture(session, 'glassdoor-sync', async (page) => {
|
|
164
168
|
// Configure viewport and user-agent to mimic a real browser
|
|
@@ -255,9 +255,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime {
|
|
|
255
255
|
let pageToken: string | undefined;
|
|
256
256
|
let nextSyncToken: string | undefined;
|
|
257
257
|
|
|
258
|
-
|
|
258
|
+
// Safety bound — at 250 events/page, 200 pages = 50k events, more than
|
|
259
|
+
// any reasonable calendar window. Stops a runaway loop if the upstream
|
|
260
|
+
// ever returns a self-referential page token.
|
|
261
|
+
const MAX_PAGES = 200;
|
|
262
|
+
let pages = 0;
|
|
263
|
+
|
|
264
|
+
while (pages < MAX_PAGES) {
|
|
265
|
+
pages++;
|
|
266
|
+
// Always request a full page — `maxResults` is a soft cap on *stored*
|
|
267
|
+
// events, not a reason to shrink the request size (shrinking to 1 once the
|
|
268
|
+
// cap is hit would crawl a busy calendar one event per round-trip).
|
|
259
269
|
const params = new URLSearchParams({
|
|
260
|
-
maxResults:
|
|
270
|
+
maxResults: '250',
|
|
261
271
|
orderBy: 'startTime',
|
|
262
272
|
singleEvents: 'true',
|
|
263
273
|
timeMin: timeMin.toISOString(),
|
|
@@ -280,6 +290,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime {
|
|
|
280
290
|
|
|
281
291
|
if (data.items) {
|
|
282
292
|
for (const calEvent of data.items) {
|
|
293
|
+
if (events.length >= maxResults) break;
|
|
283
294
|
const envelope = this.calendarEventToEnvelope(calEvent);
|
|
284
295
|
if (envelope) events.push(envelope);
|
|
285
296
|
}
|
|
@@ -287,7 +298,12 @@ export default class GoogleCalendarConnector extends ConnectorRuntime {
|
|
|
287
298
|
|
|
288
299
|
nextSyncToken = data.nextSyncToken;
|
|
289
300
|
pageToken = data.nextPageToken;
|
|
290
|
-
|
|
301
|
+
// Google only returns nextSyncToken on the LAST page (no nextPageToken).
|
|
302
|
+
// Must keep paginating until pageToken is exhausted, otherwise the sync
|
|
303
|
+
// token is never obtained and every subsequent sync re-runs the full
|
|
304
|
+
// window from scratch — so we keep paging past `maxResults`, just stop
|
|
305
|
+
// appending events once the cap is reached.
|
|
306
|
+
if (!pageToken) break;
|
|
291
307
|
}
|
|
292
308
|
|
|
293
309
|
return this.buildResult(events, nextSyncToken, events.length);
|
|
@@ -341,9 +357,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime {
|
|
|
341
357
|
let pageToken: string | undefined;
|
|
342
358
|
let nextSyncToken: string | undefined;
|
|
343
359
|
|
|
344
|
-
|
|
360
|
+
// Same hard ceiling as the full-sync path — defensive only.
|
|
361
|
+
const MAX_PAGES = 200;
|
|
362
|
+
let pages = 0;
|
|
363
|
+
|
|
364
|
+
while (pages < MAX_PAGES) {
|
|
365
|
+
pages++;
|
|
345
366
|
const params = new URLSearchParams({
|
|
346
|
-
maxResults: String(Math.min(250, maxResults - events.length)),
|
|
367
|
+
maxResults: String(Math.max(1, Math.min(250, maxResults - events.length))),
|
|
347
368
|
syncToken,
|
|
348
369
|
});
|
|
349
370
|
if (pageToken) {
|
|
@@ -375,7 +396,8 @@ export default class GoogleCalendarConnector extends ConnectorRuntime {
|
|
|
375
396
|
|
|
376
397
|
nextSyncToken = data.nextSyncToken;
|
|
377
398
|
pageToken = data.nextPageToken;
|
|
378
|
-
|
|
399
|
+
// Paginate until exhausted so we capture the trailing nextSyncToken.
|
|
400
|
+
if (!pageToken) break;
|
|
379
401
|
}
|
|
380
402
|
|
|
381
403
|
return { events, nextSyncToken };
|
|
@@ -135,7 +135,7 @@ export default class GmailConnector extends ConnectorRuntime {
|
|
|
135
135
|
},
|
|
136
136
|
entityLinks: [
|
|
137
137
|
{
|
|
138
|
-
entityType: '
|
|
138
|
+
entityType: 'person',
|
|
139
139
|
autoCreate: true,
|
|
140
140
|
titlePath: 'metadata.from_name',
|
|
141
141
|
identities: [{ namespace: IDENTITY.EMAIL, eventPath: 'metadata.from_email' }],
|
|
@@ -274,8 +274,11 @@ export default class GmailConnector extends ConnectorRuntime {
|
|
|
274
274
|
return d;
|
|
275
275
|
})();
|
|
276
276
|
|
|
277
|
-
|
|
278
|
-
|
|
277
|
+
// Gmail's `after:` accepts a Unix timestamp (epoch seconds) for second-level
|
|
278
|
+
// precision. Using `YYYY/MM/DD` (day granularity, host timezone) meant every
|
|
279
|
+
// sync within the same day re-fetched the whole day's threads as duplicates.
|
|
280
|
+
const afterEpochSeconds = Math.floor(afterDate.getTime() / 1000);
|
|
281
|
+
const query = `after:${afterEpochSeconds} label:${label}`;
|
|
279
282
|
|
|
280
283
|
const events: EventEnvelope[] = [];
|
|
281
284
|
let pageToken: string | undefined;
|
|
@@ -73,9 +73,16 @@ interface RawReview {
|
|
|
73
73
|
*/
|
|
74
74
|
function parseDate(dateArray: unknown): string | null {
|
|
75
75
|
if (!Array.isArray(dateArray)) return null;
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
// Compute numerically: seconds*1000 + millis. The previous string-concat
|
|
77
|
+
// approach (`${seconds}${millis}`) only worked when millis was a 3-digit
|
|
78
|
+
// zero-padded string; Google sends a plain integer, so e.g. `[s, 5]` produced
|
|
79
|
+
// a date in 1970 and `[s, 50]` a date in year ~7340.
|
|
80
|
+
const seconds = Number(dateArray[0]);
|
|
81
|
+
const millis = Number(dateArray[1] ?? 0);
|
|
82
|
+
if (!Number.isFinite(seconds) || !Number.isFinite(millis)) return null;
|
|
83
|
+
const d = new Date(seconds * 1000 + millis);
|
|
84
|
+
if (Number.isNaN(d.getTime())) return null;
|
|
85
|
+
return d.toJSON();
|
|
79
86
|
}
|
|
80
87
|
|
|
81
88
|
/**
|
|
@@ -129,6 +136,12 @@ async function fetchReviewsPage(
|
|
|
129
136
|
|
|
130
137
|
if (!res.ok) {
|
|
131
138
|
if (res.status === 404) throw new Error('App not found (404)');
|
|
139
|
+
if (res.status === 429) {
|
|
140
|
+
const retryAfter = res.headers.get('Retry-After');
|
|
141
|
+
throw new Error(
|
|
142
|
+
`Google Play rate limit (429). Retry after ${retryAfter ?? 'unknown'} seconds.`
|
|
143
|
+
);
|
|
144
|
+
}
|
|
132
145
|
throw new Error(`Google Play request failed: ${res.status} ${res.statusText}`);
|
|
133
146
|
}
|
|
134
147
|
|
|
@@ -136,14 +149,28 @@ async function fetchReviewsPage(
|
|
|
136
149
|
|
|
137
150
|
// Response starts with ")]}'" (security prefix), then a newline, then JSON.
|
|
138
151
|
// The library skips the first 5 characters.
|
|
139
|
-
|
|
152
|
+
// Wrap parse in try/catch — Google sometimes returns an HTML interstitial
|
|
153
|
+
// (captcha / geo-block / maintenance) with status 200, which would bubble up
|
|
154
|
+
// as an unhelpful SyntaxError otherwise.
|
|
155
|
+
let outer: any;
|
|
156
|
+
try {
|
|
157
|
+
outer = JSON.parse(text.substring(5));
|
|
158
|
+
} catch {
|
|
159
|
+
const preview = text.substring(0, 120).replace(/\s+/g, ' ');
|
|
160
|
+
throw new Error(`Google Play returned non-JSON response: ${preview}`);
|
|
161
|
+
}
|
|
140
162
|
const innerJson: string | null = outer?.[0]?.[2];
|
|
141
163
|
|
|
142
164
|
if (innerJson === null || innerJson === undefined) {
|
|
143
165
|
return { reviews: [], nextToken: null };
|
|
144
166
|
}
|
|
145
167
|
|
|
146
|
-
|
|
168
|
+
let data: any;
|
|
169
|
+
try {
|
|
170
|
+
data = JSON.parse(innerJson);
|
|
171
|
+
} catch {
|
|
172
|
+
throw new Error('Google Play returned malformed inner JSON payload');
|
|
173
|
+
}
|
|
147
174
|
return {
|
|
148
175
|
reviews: extractReviews(data, appId),
|
|
149
176
|
nextToken: extractPaginationToken(data),
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
type SyncContext,
|
|
17
17
|
type SyncResult,
|
|
18
18
|
} from '@lobu/connector-sdk';
|
|
19
|
+
import { validatePublicUrl } from './browser-scraper-utils.ts';
|
|
19
20
|
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
21
22
|
// Algolia HN API types
|
|
@@ -261,11 +262,36 @@ export default class HackerNewsConnector extends ConnectorRuntime {
|
|
|
261
262
|
`&numericFilters=${encodeURIComponent(`created_at_i>${lookbackTimestamp}`)}`;
|
|
262
263
|
|
|
263
264
|
const response = await fetch(url);
|
|
265
|
+
|
|
266
|
+
// Honor Algolia's rate-limit response so we don't hammer them and turn
|
|
267
|
+
// a transient 429 into "Unexpected token < in JSON" when the next call
|
|
268
|
+
// returns an HTML error page.
|
|
269
|
+
if (response.status === 429) {
|
|
270
|
+
const retryAfter = response.headers.get('Retry-After');
|
|
271
|
+
const waitMs = retryAfter ? Math.min(60_000, Math.max(1, Number(retryAfter)) * 1000) : 5000;
|
|
272
|
+
await this.sleep(Number.isFinite(waitMs) ? waitMs : 5000);
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
|
|
264
276
|
if (!response.ok) {
|
|
265
|
-
|
|
277
|
+
const text = await response.text().catch(() => '');
|
|
278
|
+
throw new Error(`Algolia API error (${response.status}): ${text}`);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Algolia normally returns JSON, but proxies/captive portals occasionally
|
|
282
|
+
// return HTML. Surface a useful error instead of a bare SyntaxError that
|
|
283
|
+
// makes the connector look broken when the upstream is at fault.
|
|
284
|
+
let data: AlgoliaResponse;
|
|
285
|
+
try {
|
|
286
|
+
data = (await response.json()) as AlgoliaResponse;
|
|
287
|
+
} catch (err) {
|
|
288
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
289
|
+
throw new Error(`Algolia API returned non-JSON response: ${message}`);
|
|
266
290
|
}
|
|
267
291
|
|
|
268
|
-
|
|
292
|
+
if (!data || !Array.isArray(data.hits)) {
|
|
293
|
+
throw new Error('Algolia API returned an unexpected response shape');
|
|
294
|
+
}
|
|
269
295
|
|
|
270
296
|
for (const hit of data.hits) {
|
|
271
297
|
if (contentType === 'comment') {
|
|
@@ -404,6 +430,15 @@ export default class HackerNewsConnector extends ConnectorRuntime {
|
|
|
404
430
|
|
|
405
431
|
private async fetchExternalContent(url: string): Promise<string | null> {
|
|
406
432
|
try {
|
|
433
|
+
// SSRF guard — `url` is supplied by whoever submitted the HN story and
|
|
434
|
+
// is therefore attacker-controllable. Refuse to fetch private/internal
|
|
435
|
+
// addresses (loopback, 169.254.169.254 cloud metadata, RFC1918, etc.).
|
|
436
|
+
try {
|
|
437
|
+
validatePublicUrl(url);
|
|
438
|
+
} catch {
|
|
439
|
+
return null;
|
|
440
|
+
}
|
|
441
|
+
|
|
407
442
|
const controller = new AbortController();
|
|
408
443
|
const timeoutId = setTimeout(() => controller.abort(), this.CONTENT_FETCH_TIMEOUT);
|
|
409
444
|
|
package/dist/connectors/index.ts
CHANGED
|
@@ -1,12 +1,23 @@
|
|
|
1
|
+
export * from './apple_health.ts';
|
|
2
|
+
export * from './apple_photos.ts';
|
|
3
|
+
export * from './apple_screen_time.ts';
|
|
4
|
+
export * from './local_directory.ts';
|
|
1
5
|
export * from './browser-scraper-utils.ts';
|
|
6
|
+
// Browser primitives — connector definitions whose executors live in the
|
|
7
|
+
// Owletto for Chrome extension (apps/chrome/executor.js). Kept under
|
|
8
|
+
// browser/ so they're structurally distinct from third-party service
|
|
9
|
+
// connectors (linkedin, revolut, github, etc.).
|
|
10
|
+
export * from './browser/evaluate.ts';
|
|
11
|
+
export * from './browser/fill_form.ts';
|
|
12
|
+
export * from './browser/page_text.ts';
|
|
2
13
|
export * from './capterra.ts';
|
|
14
|
+
export * from './chrome_tabs.ts';
|
|
3
15
|
export * from './g2.ts';
|
|
4
16
|
export * from './github.ts';
|
|
5
17
|
export * from './glassdoor.ts';
|
|
6
18
|
export * from './gmaps.ts';
|
|
7
19
|
export * from './google_calendar.ts';
|
|
8
20
|
export * from './google_gmail.ts';
|
|
9
|
-
export * from './google_photos.ts';
|
|
10
21
|
export * from './google_play.ts';
|
|
11
22
|
export * from './hackernews.ts';
|
|
12
23
|
export * from './ios_appstore.ts';
|
|
@@ -14,10 +25,12 @@ export * from './linkedin.ts';
|
|
|
14
25
|
export * from './microsoft_outlook.ts';
|
|
15
26
|
export * from './producthunt.ts';
|
|
16
27
|
export * from './reddit.ts';
|
|
28
|
+
export * from './revolut.ts';
|
|
17
29
|
export * from './rss.ts';
|
|
18
30
|
export * from './spotify.ts';
|
|
19
31
|
export * from './trustpilot.ts';
|
|
20
32
|
export * from './website.ts';
|
|
21
33
|
export * from './whatsapp.ts';
|
|
34
|
+
export * from './whatsapp_local.ts';
|
|
22
35
|
export * from './x.ts';
|
|
23
36
|
export * from './youtube.ts';
|
|
@@ -19,7 +19,12 @@ import {
|
|
|
19
19
|
type SyncContext,
|
|
20
20
|
type SyncResult,
|
|
21
21
|
} from '@lobu/connector-sdk';
|
|
22
|
-
import {
|
|
22
|
+
import {
|
|
23
|
+
getBrowserCdpUrl,
|
|
24
|
+
getBrowserCookies,
|
|
25
|
+
getBrowserUserDataDir,
|
|
26
|
+
validateCookieNotExpired,
|
|
27
|
+
} from './browser-scraper-utils';
|
|
23
28
|
|
|
24
29
|
// ── Types ──────────────────────────────────────────────────────
|
|
25
30
|
|
|
@@ -316,23 +321,37 @@ export default class LinkedInConnector extends ConnectorRuntime {
|
|
|
316
321
|
// Normalize URL - remove trailing slash
|
|
317
322
|
const baseUrl = companyUrl.replace(/\/$/, '');
|
|
318
323
|
|
|
319
|
-
const
|
|
320
|
-
|
|
324
|
+
const userDataDir = getBrowserUserDataDir(ctx.sessionState);
|
|
325
|
+
const cdpUrlFromSession = getBrowserCdpUrl(ctx.sessionState);
|
|
326
|
+
const cdpUrl = cdpUrlFromSession ?? 'auto';
|
|
327
|
+
// No need to require cookies when the device tells us to attach directly
|
|
328
|
+
// (managed --user-data-dir on disk, or an explicit CDP endpoint pointed
|
|
329
|
+
// at the user's running Chrome). The cookie cascade is only the fallback
|
|
330
|
+
// for the cloud/auto path.
|
|
331
|
+
const skipServerCookies = !!userDataDir || !!cdpUrlFromSession;
|
|
332
|
+
const cookies = skipServerCookies
|
|
333
|
+
? []
|
|
334
|
+
: getBrowserCookies(ctx.checkpoint as any, ctx.sessionState as any, 'linkedin');
|
|
335
|
+
if (!skipServerCookies) {
|
|
336
|
+
validateCookieNotExpired(cookies, 'li_at', 'linkedin');
|
|
337
|
+
}
|
|
321
338
|
|
|
322
339
|
const maxScrolls = (config.max_scrolls as number) ?? (feedKey === 'jobs' ? 3 : 5);
|
|
323
340
|
|
|
324
341
|
if (feedKey === 'jobs') {
|
|
325
|
-
return this.syncJobs(baseUrl, cookies, maxScrolls, checkpoint);
|
|
342
|
+
return this.syncJobs(baseUrl, cookies, maxScrolls, checkpoint, userDataDir, cdpUrl);
|
|
326
343
|
}
|
|
327
344
|
|
|
328
|
-
return this.syncUpdates(baseUrl, cookies, maxScrolls, checkpoint);
|
|
345
|
+
return this.syncUpdates(baseUrl, cookies, maxScrolls, checkpoint, userDataDir, cdpUrl);
|
|
329
346
|
}
|
|
330
347
|
|
|
331
348
|
private async syncUpdates(
|
|
332
349
|
baseUrl: string,
|
|
333
350
|
cookies: any[],
|
|
334
351
|
maxScrolls: number,
|
|
335
|
-
checkpoint: LinkedInCheckpoint
|
|
352
|
+
checkpoint: LinkedInCheckpoint,
|
|
353
|
+
userDataDir: string | undefined,
|
|
354
|
+
cdpUrl: string | 'auto'
|
|
336
355
|
): Promise<SyncResult> {
|
|
337
356
|
const postsUrl = `${baseUrl}/posts/`;
|
|
338
357
|
|
|
@@ -350,8 +369,9 @@ export default class LinkedInConnector extends ConnectorRuntime {
|
|
|
350
369
|
navigationTimeoutMs: 20000,
|
|
351
370
|
},
|
|
352
371
|
url: postsUrl,
|
|
353
|
-
cdpUrl
|
|
372
|
+
cdpUrl,
|
|
354
373
|
cookies,
|
|
374
|
+
userDataDir,
|
|
355
375
|
parseResponse: parseCompanyUpdates,
|
|
356
376
|
checkAuth: async (page) => {
|
|
357
377
|
const url = page.url();
|
|
@@ -401,7 +421,9 @@ export default class LinkedInConnector extends ConnectorRuntime {
|
|
|
401
421
|
baseUrl: string,
|
|
402
422
|
cookies: any[],
|
|
403
423
|
maxScrolls: number,
|
|
404
|
-
checkpoint: LinkedInCheckpoint
|
|
424
|
+
checkpoint: LinkedInCheckpoint,
|
|
425
|
+
userDataDir: string | undefined,
|
|
426
|
+
cdpUrl: string | 'auto'
|
|
405
427
|
): Promise<SyncResult> {
|
|
406
428
|
const jobsUrl = `${baseUrl}/jobs/`;
|
|
407
429
|
|
|
@@ -420,8 +442,9 @@ export default class LinkedInConnector extends ConnectorRuntime {
|
|
|
420
442
|
navigationTimeoutMs: 20000,
|
|
421
443
|
},
|
|
422
444
|
url: jobsUrl,
|
|
423
|
-
cdpUrl
|
|
445
|
+
cdpUrl,
|
|
424
446
|
cookies,
|
|
447
|
+
userDataDir,
|
|
425
448
|
parseResponse: parseJobListings,
|
|
426
449
|
checkAuth: async (page) => {
|
|
427
450
|
const url = page.url();
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local Directory Connector (V1 runtime) — Lobu for Mac only.
|
|
3
|
+
*
|
|
4
|
+
* Syncs text files (txt/md/json/csv/html) from a local folder on the user's
|
|
5
|
+
* Mac via Lobu for Mac. The app advertises the `local_directory`
|
|
6
|
+
* capability on /api/workers/poll, reads the folder, and streams file events
|
|
7
|
+
* back through the standard worker protocol.
|
|
8
|
+
*
|
|
9
|
+
* The sync() / execute() stubs here throw immediately if a server-side worker
|
|
10
|
+
* somehow bypassed the capability gate — same pattern as apple_screen_time.ts.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
type ActionResult,
|
|
15
|
+
type ConnectorDefinition,
|
|
16
|
+
ConnectorRuntime,
|
|
17
|
+
type SyncContext,
|
|
18
|
+
type SyncResult,
|
|
19
|
+
} from '@lobu/connector-sdk';
|
|
20
|
+
|
|
21
|
+
const BRIDGE_ONLY_MESSAGE =
|
|
22
|
+
'local.directory runs only on a worker advertising capability "local_directory" (Lobu for Mac). ' +
|
|
23
|
+
'This run was claimed by a worker without that capability — check connector_definitions.required_capability and the poll-time capability filter.';
|
|
24
|
+
|
|
25
|
+
export default class LocalDirectoryConnector extends ConnectorRuntime {
|
|
26
|
+
readonly definition: ConnectorDefinition = {
|
|
27
|
+
key: 'local.directory',
|
|
28
|
+
name: 'Local Folder',
|
|
29
|
+
description:
|
|
30
|
+
'Sync text files (txt/md/json/csv/html) from a folder on your Mac via Lobu for Mac.',
|
|
31
|
+
version: '0.1.0',
|
|
32
|
+
faviconDomain: 'apple.com',
|
|
33
|
+
requiredCapability: 'local_directory',
|
|
34
|
+
runtime: { platforms: ['macos'] },
|
|
35
|
+
authSchema: { methods: [{ type: 'none' }] },
|
|
36
|
+
feeds: {
|
|
37
|
+
files: {
|
|
38
|
+
key: 'files',
|
|
39
|
+
name: 'Files',
|
|
40
|
+
description: 'Text files from one local folder on the user\'s Mac. One feed per folder — folder_id is an opaque stable id minted by the Mac app (the security-scoped bookmark is held device-side; the server never sees the absolute path).',
|
|
41
|
+
userManaged: true,
|
|
42
|
+
configSchema: {
|
|
43
|
+
type: 'object',
|
|
44
|
+
required: ['folder_id', 'display_name'],
|
|
45
|
+
properties: {
|
|
46
|
+
folder_id: {
|
|
47
|
+
type: 'string',
|
|
48
|
+
minLength: 8,
|
|
49
|
+
maxLength: 64,
|
|
50
|
+
description: 'Opaque stable id (UUID) minted on the Mac. Maps to a security-scoped bookmark stored locally on the device.',
|
|
51
|
+
},
|
|
52
|
+
display_name: {
|
|
53
|
+
type: 'string',
|
|
54
|
+
minLength: 1,
|
|
55
|
+
maxLength: 200,
|
|
56
|
+
description: 'Folder name shown in the UI (e.g., "Documents"). Not used to locate the folder — the device resolves folder_id to its bookmark.',
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
eventKinds: {
|
|
61
|
+
file_document: {
|
|
62
|
+
description: 'A text file from a configured local folder.',
|
|
63
|
+
metadataSchema: {
|
|
64
|
+
type: 'object',
|
|
65
|
+
// No absolute filesystem path — the bridge sends the folder's
|
|
66
|
+
// display name and the file name, which is enough context
|
|
67
|
+
// without leaking the user's home directory / disk layout.
|
|
68
|
+
required: ['source', 'folder', 'name'],
|
|
69
|
+
properties: {
|
|
70
|
+
source: { type: 'string', const: 'local_directory' },
|
|
71
|
+
folder: { type: 'string', description: 'Display name of the local folder.' },
|
|
72
|
+
name: { type: 'string', description: 'File name.' },
|
|
73
|
+
ext: { type: 'string' },
|
|
74
|
+
size_bytes: { type: 'number' },
|
|
75
|
+
modified_at: { type: 'string' },
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
async sync(_ctx: SyncContext): Promise<SyncResult> {
|
|
85
|
+
throw new Error(BRIDGE_ONLY_MESSAGE);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async execute(): Promise<ActionResult> {
|
|
89
|
+
throw new Error(BRIDGE_ONLY_MESSAGE);
|
|
90
|
+
}
|
|
91
|
+
}
|