@lobu/cli 6.1.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/_lib/apply/apply-cmd.d.ts +36 -0
- package/dist/commands/_lib/apply/apply-cmd.d.ts.map +1 -1
- package/dist/commands/_lib/apply/apply-cmd.js +696 -40
- package/dist/commands/_lib/apply/apply-cmd.js.map +1 -1
- package/dist/commands/_lib/apply/client.d.ts +285 -0
- package/dist/commands/_lib/apply/client.d.ts.map +1 -1
- package/dist/commands/_lib/apply/client.js +469 -28
- package/dist/commands/_lib/apply/client.js.map +1 -1
- package/dist/commands/_lib/apply/desired-state.d.ts +187 -3
- package/dist/commands/_lib/apply/desired-state.d.ts.map +1 -1
- package/dist/commands/_lib/apply/desired-state.js +879 -88
- package/dist/commands/_lib/apply/desired-state.js.map +1 -1
- package/dist/commands/_lib/apply/diff.d.ts +72 -3
- package/dist/commands/_lib/apply/diff.d.ts.map +1 -1
- package/dist/commands/_lib/apply/diff.js +473 -84
- package/dist/commands/_lib/apply/diff.js.map +1 -1
- package/dist/commands/_lib/apply/prompt.d.ts +6 -0
- package/dist/commands/_lib/apply/prompt.d.ts.map +1 -1
- package/dist/commands/_lib/apply/prompt.js +16 -0
- package/dist/commands/_lib/apply/prompt.js.map +1 -1
- package/dist/commands/_lib/apply/render.d.ts +9 -0
- package/dist/commands/_lib/apply/render.d.ts.map +1 -1
- package/dist/commands/_lib/apply/render.js +80 -3
- package/dist/commands/_lib/apply/render.js.map +1 -1
- package/dist/commands/_lib/connector-loader.d.ts +3 -0
- package/dist/commands/_lib/connector-loader.d.ts.map +1 -0
- package/dist/commands/_lib/connector-loader.js +129 -0
- package/dist/commands/_lib/connector-loader.js.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts +35 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.js +351 -0
- package/dist/commands/_lib/connector-run-cmd.js.map +1 -0
- package/dist/commands/_lib/export/export-cmd.d.ts +35 -0
- package/dist/commands/_lib/export/export-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/export/export-cmd.js +329 -0
- package/dist/commands/_lib/export/export-cmd.js.map +1 -0
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +11 -14
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/chat.d.ts.map +1 -1
- package/dist/commands/chat.js +28 -7
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/connector.d.ts +3 -0
- package/dist/commands/connector.d.ts.map +1 -0
- package/dist/commands/connector.js +5 -0
- package/dist/commands/connector.js.map +1 -0
- package/dist/commands/dev.d.ts +23 -0
- package/dist/commands/dev.d.ts.map +1 -1
- package/dist/commands/dev.js +273 -8
- package/dist/commands/dev.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +2 -3
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +28 -18
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/init.d.ts +2 -0
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +29 -1
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/login.d.ts.map +1 -1
- package/dist/commands/login.js +22 -16
- package/dist/commands/login.js.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.d.ts.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.js +15 -144
- package/dist/commands/memory/_lib/browser-auth-cmd.js.map +1 -1
- package/dist/commands/memory/_lib/schema.d.ts +28 -1
- package/dist/commands/memory/_lib/schema.d.ts.map +1 -1
- package/dist/commands/memory/_lib/schema.js +120 -4
- package/dist/commands/memory/_lib/schema.js.map +1 -1
- package/dist/commands/memory/_lib/seed-cmd.d.ts.map +1 -1
- package/dist/commands/memory/_lib/seed-cmd.js +41 -18
- package/dist/commands/memory/_lib/seed-cmd.js.map +1 -1
- package/dist/commands/org.d.ts +4 -0
- package/dist/commands/org.d.ts.map +1 -1
- package/dist/commands/org.js +10 -0
- package/dist/commands/org.js.map +1 -1
- package/dist/commands/token.d.ts +9 -0
- package/dist/commands/token.d.ts.map +1 -1
- package/dist/commands/token.js +54 -3
- package/dist/commands/token.js.map +1 -1
- package/dist/commands/validate.d.ts.map +1 -1
- package/dist/commands/validate.js +4 -13
- package/dist/commands/validate.js.map +1 -1
- package/dist/config/loader.js +2 -2
- package/dist/config/loader.js.map +1 -1
- package/dist/connectors/README.md +2 -3
- package/dist/connectors/apple_health.ts +138 -0
- package/dist/connectors/apple_photos.ts +178 -0
- package/dist/connectors/apple_screen_time.ts +82 -0
- package/dist/connectors/browser/evaluate.ts +120 -0
- package/dist/connectors/browser/fill_form.ts +107 -0
- package/dist/connectors/browser/page_text.ts +108 -0
- package/dist/connectors/browser-scraper-utils.ts +111 -3
- package/dist/connectors/capterra.ts +5 -1
- package/dist/connectors/chrome_tabs.ts +74 -0
- package/dist/connectors/g2.ts +5 -1
- package/dist/connectors/github.ts +16 -38
- package/dist/connectors/glassdoor.ts +5 -1
- package/dist/connectors/google_calendar.ts +28 -6
- package/dist/connectors/google_gmail.ts +6 -3
- package/dist/connectors/google_play.ts +32 -5
- package/dist/connectors/hackernews.ts +37 -2
- package/dist/connectors/index.ts +14 -1
- package/dist/connectors/linkedin.ts +32 -9
- package/dist/connectors/local_directory.ts +91 -0
- package/dist/connectors/reddit.ts +1 -0
- package/dist/connectors/revolut.ts +569 -0
- package/dist/connectors/rss.ts +33 -8
- package/dist/connectors/trustpilot.ts +36 -21
- package/dist/connectors/website.ts +8 -69
- package/dist/connectors/whatsapp.ts +21 -22
- package/dist/connectors/whatsapp_local.ts +125 -0
- package/dist/connectors/x.ts +17 -7
- package/dist/db/migrations/20260510220000_connector_required_capability.sql +47 -0
- package/dist/db/migrations/20260512000000_device_worker_connection_binding.sql +113 -0
- package/dist/db/migrations/20260512131703_connections_slug.sql +131 -0
- package/dist/db/migrations/20260513000000_chat_user_identities.sql +24 -0
- package/dist/db/migrations/20260513120000_auth_profiles_device_binding.sql +50 -0
- package/dist/db/migrations/20260513150000_auth_profiles_cdp_url.sql +43 -0
- package/dist/db/migrations/20260513200000_notifications_as_events.sql +86 -0
- package/dist/db/migrations/20260514000000_scheduled_jobs.sql +97 -0
- package/dist/db/migrations/20260514120000_auth_profiles_connector_key_nullable.sql +42 -0
- package/dist/db/migrations/20260514130000_connection_action_modes.sql +103 -0
- package/dist/db/migrations/20260514160000_auth_profiles_mirror_mode.sql +32 -0
- package/dist/db/migrations/20260515120000_agents_per_org_pk.sql +66 -0
- package/dist/db/migrations/20260515150000_geo_enrichment.sql +208 -0
- package/dist/db/migrations/20260515160000_drop_agents_org_id_unique.sql +24 -0
- package/dist/db/migrations/20260515170000_auth_profiles_default_for_connector.sql +23 -0
- package/dist/db/migrations/20260516120000_agents_per_org_pk_swap.sql +125 -0
- package/dist/db/migrations/20260516200000_events_search_tsv.sql +134 -0
- package/dist/db/migrations/20260516200100_events_lifecycle_changes_index.sql +25 -0
- package/dist/db/migrations/20260517010000_drop_unused_indexes.sql +49 -0
- package/dist/db/migrations/20260517020000_softdelete_orphan_feeds.sql +56 -0
- package/dist/db/migrations/20260517030000_pat_worker_id_binding.sql +27 -0
- package/dist/db/migrations/20260517040000_archive_orphan_watchers.sql +30 -0
- package/dist/db/migrations/20260517050000_watcher_agent_id_not_null.sql +34 -0
- package/dist/db/migrations/20260517060000_watcher_schema_additions.sql +78 -0
- package/dist/db/migrations/20260517150000_goals_primitive.sql +55 -0
- package/dist/db/migrations/20260517160000_drop_goals_primitive.sql +45 -0
- package/dist/db/migrations/20260518000000_pending_interactions.sql +49 -0
- package/dist/db/migrations/20260518010000_runs_heartbeat_reaper_index.sql +22 -0
- package/dist/eval/client.d.ts.map +1 -1
- package/dist/eval/client.js +11 -0
- package/dist/eval/client.js.map +1 -1
- package/dist/eval/grader.js +2 -1
- package/dist/eval/grader.js.map +1 -1
- package/dist/eval/types.d.ts +2 -0
- package/dist/eval/types.d.ts.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +115 -114
- package/dist/index.js.map +1 -1
- package/dist/internal/context.d.ts +9 -0
- package/dist/internal/context.d.ts.map +1 -1
- package/dist/internal/context.js +41 -6
- package/dist/internal/context.js.map +1 -1
- package/dist/internal/credentials.d.ts +5 -0
- package/dist/internal/credentials.d.ts.map +1 -1
- package/dist/internal/credentials.js +75 -1
- package/dist/internal/credentials.js.map +1 -1
- package/dist/internal/gateway-url.d.ts +14 -0
- package/dist/internal/gateway-url.d.ts.map +1 -1
- package/dist/internal/gateway-url.js +19 -0
- package/dist/internal/gateway-url.js.map +1 -1
- package/dist/internal/index.d.ts +1 -1
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +1 -1
- package/dist/internal/index.js.map +1 -1
- package/dist/internal/local-env.d.ts.map +1 -1
- package/dist/internal/local-env.js +9 -2
- package/dist/internal/local-env.js.map +1 -1
- package/dist/server.bundle.mjs +42251 -36931
- package/dist/start-local.bundle.mjs +16437 -9882
- package/dist/templates/TESTING.md.tmpl +9 -9
- package/package.json +8 -6
- package/dist/connectors/google_photos.ts +0 -776
|
@@ -1,776 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Google Photos Connector
|
|
3
|
-
*
|
|
4
|
-
* Reliable photo ingestion without browser scraping.
|
|
5
|
-
*
|
|
6
|
-
* 1. CDP grab (once per run): connect to the user's running Chrome and pull
|
|
7
|
-
* cookies via Network.getAllCookies at browser level. No tab attach, no
|
|
8
|
-
* page navigation — just cookies.
|
|
9
|
-
* 2. Extract WIZ tokens (FdrFJe / cfb2h / SNlM0e) from photos.google.com HTML.
|
|
10
|
-
* 3. Paginate the timeline with direct POSTs to the `lcxiM` batchexecute RPC
|
|
11
|
-
* (≈300 photos per call — GPS and place name included in the response).
|
|
12
|
-
* 4. Parallel fill of EXIF for each photo via direct POSTs to `fDcn4b`.
|
|
13
|
-
*
|
|
14
|
-
* Requires Chrome running with remote debugging enabled on the worker host.
|
|
15
|
-
*/
|
|
16
|
-
import { createHash } from 'node:crypto';
|
|
17
|
-
import {
|
|
18
|
-
type ActionContext,
|
|
19
|
-
type ActionResult,
|
|
20
|
-
type ConnectorDefinition,
|
|
21
|
-
ConnectorRuntime,
|
|
22
|
-
type EventEnvelope,
|
|
23
|
-
resolveCdpUrl,
|
|
24
|
-
type SyncContext,
|
|
25
|
-
type SyncResult,
|
|
26
|
-
sdkLogger,
|
|
27
|
-
} from '@lobu/connector-sdk';
|
|
28
|
-
|
|
29
|
-
// ── Types ──────────────────────────────────────────────────────
|
|
30
|
-
|
|
31
|
-
interface GooglePhotosCheckpoint {
|
|
32
|
-
last_photo_id?: string;
|
|
33
|
-
last_timestamp?: string;
|
|
34
|
-
account_fingerprint?: string;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
interface WizSession {
|
|
38
|
-
sid: string;
|
|
39
|
-
bl: string;
|
|
40
|
-
at: string;
|
|
41
|
-
cookieHeader: string;
|
|
42
|
-
accountFingerprint: string;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
interface TimelinePhoto {
|
|
46
|
-
mediaKey: string;
|
|
47
|
-
cdnUrl: string;
|
|
48
|
-
width: number;
|
|
49
|
-
height: number;
|
|
50
|
-
dateTaken: Date;
|
|
51
|
-
modifiedAt?: Date;
|
|
52
|
-
latitude?: number;
|
|
53
|
-
longitude?: number;
|
|
54
|
-
locationName?: string;
|
|
55
|
-
placeId?: string;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
interface PhotoDetail {
|
|
59
|
-
filename?: string;
|
|
60
|
-
cameraMake?: string;
|
|
61
|
-
cameraModel?: string;
|
|
62
|
-
focalOrStop?: number;
|
|
63
|
-
aperture?: number;
|
|
64
|
-
iso?: number;
|
|
65
|
-
shutterSec?: number;
|
|
66
|
-
tzOffsetMin?: number;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// ── CDP cookie grab ────────────────────────────────────────────
|
|
70
|
-
|
|
71
|
-
const COOKIE_DOMAIN_SUFFIXES = ['google.com', 'googleusercontent.com'];
|
|
72
|
-
|
|
73
|
-
async function fetchCookiesViaCdp(cdpUrl: string): Promise<string> {
|
|
74
|
-
const wsUrl = cdpUrl.startsWith('ws') ? cdpUrl : await resolveToWs(cdpUrl);
|
|
75
|
-
sdkLogger.info({ wsUrl }, '[GooglePhotos] Opening CDP for cookie grab');
|
|
76
|
-
|
|
77
|
-
const ws = new WebSocket(wsUrl);
|
|
78
|
-
await new Promise<void>((resolve, reject) => {
|
|
79
|
-
const t = setTimeout(() => reject(new Error('CDP connect timeout')), 60000);
|
|
80
|
-
ws.onopen = () => {
|
|
81
|
-
clearTimeout(t);
|
|
82
|
-
resolve();
|
|
83
|
-
};
|
|
84
|
-
ws.onerror = () => {
|
|
85
|
-
clearTimeout(t);
|
|
86
|
-
reject(new Error('CDP connect failed — is Chrome running with --remote-debugging-port?'));
|
|
87
|
-
};
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
let msgId = 1;
|
|
91
|
-
const sendBrowser = (method: string, params: Record<string, unknown> = {}): Promise<any> =>
|
|
92
|
-
new Promise((resolve, reject) => {
|
|
93
|
-
const id = msgId++;
|
|
94
|
-
const t = setTimeout(() => reject(new Error(`${method} timeout`)), 30000);
|
|
95
|
-
const handler = (e: MessageEvent) => {
|
|
96
|
-
const data = JSON.parse(e.data as string);
|
|
97
|
-
if (data.id !== id) return;
|
|
98
|
-
clearTimeout(t);
|
|
99
|
-
ws.removeEventListener('message', handler);
|
|
100
|
-
data.error ? reject(new Error(data.error.message)) : resolve(data.result);
|
|
101
|
-
};
|
|
102
|
-
ws.addEventListener('message', handler);
|
|
103
|
-
ws.send(JSON.stringify({ id, method, params }));
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
try {
|
|
107
|
-
const { targetId } = await sendBrowser('Target.createTarget', { url: 'about:blank' });
|
|
108
|
-
const { sessionId } = await sendBrowser('Target.attachToTarget', { targetId, flatten: true });
|
|
109
|
-
|
|
110
|
-
const sendSession = (method: string, params: Record<string, unknown> = {}): Promise<any> =>
|
|
111
|
-
new Promise((resolve, reject) => {
|
|
112
|
-
const id = msgId++;
|
|
113
|
-
const t = setTimeout(() => reject(new Error(`${method} timeout`)), 30000);
|
|
114
|
-
const handler = (e: MessageEvent) => {
|
|
115
|
-
const data = JSON.parse(e.data as string);
|
|
116
|
-
if (data.id !== id) return;
|
|
117
|
-
clearTimeout(t);
|
|
118
|
-
ws.removeEventListener('message', handler);
|
|
119
|
-
data.error ? reject(new Error(data.error.message)) : resolve(data.result);
|
|
120
|
-
};
|
|
121
|
-
ws.addEventListener('message', handler);
|
|
122
|
-
ws.send(JSON.stringify({ id, method, params, sessionId }));
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
await sendSession('Network.enable');
|
|
126
|
-
const { cookies } = await sendSession('Network.getAllCookies');
|
|
127
|
-
await sendBrowser('Target.closeTarget', { targetId });
|
|
128
|
-
|
|
129
|
-
const host = 'photos.google.com';
|
|
130
|
-
const matched = (cookies as Array<{ name: string; value: string; domain: string }>).filter(
|
|
131
|
-
(c) => {
|
|
132
|
-
if (!c.domain) return false;
|
|
133
|
-
const d = c.domain.startsWith('.') ? c.domain.slice(1) : c.domain;
|
|
134
|
-
if (!COOKIE_DOMAIN_SUFFIXES.some((s) => d === s || d.endsWith(`.${s}`))) return false;
|
|
135
|
-
// For cookie header: include if cookie applies to photos.google.com
|
|
136
|
-
if (c.domain.startsWith('.')) {
|
|
137
|
-
const s = c.domain.slice(1);
|
|
138
|
-
return host === s || host.endsWith(`.${s}`);
|
|
139
|
-
}
|
|
140
|
-
return c.domain === host;
|
|
141
|
-
}
|
|
142
|
-
);
|
|
143
|
-
sdkLogger.info({ n: matched.length }, '[GooglePhotos] cookies captured');
|
|
144
|
-
return matched.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
145
|
-
} finally {
|
|
146
|
-
ws.close();
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
async function resolveToWs(httpUrl: string): Promise<string> {
|
|
151
|
-
const { fetchCdpVersionInfo } = await import('@lobu/connector-sdk');
|
|
152
|
-
const info = await fetchCdpVersionInfo(httpUrl);
|
|
153
|
-
if (!info?.webSocketDebuggerUrl) {
|
|
154
|
-
throw new Error(`CDP endpoint did not respond at ${httpUrl}`);
|
|
155
|
-
}
|
|
156
|
-
const parsed = new URL(httpUrl);
|
|
157
|
-
const ws = new URL(info.webSocketDebuggerUrl);
|
|
158
|
-
ws.hostname = parsed.hostname;
|
|
159
|
-
ws.port = parsed.port;
|
|
160
|
-
return ws.toString();
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// ── Session bootstrap ──────────────────────────────────────────
|
|
164
|
-
|
|
165
|
-
const USER_AGENT =
|
|
166
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36';
|
|
167
|
-
|
|
168
|
-
/**
|
|
169
|
-
* SAPISID is the per-Google-account master cookie; hashing it yields a stable
|
|
170
|
-
* identifier for detecting account drift without leaking auth material if the
|
|
171
|
-
* checkpoint is dumped.
|
|
172
|
-
*/
|
|
173
|
-
function accountFingerprintFromCookies(cookieHeader: string): string {
|
|
174
|
-
const match = cookieHeader.match(/(?:^|;\s*)SAPISID=([^;]+)/);
|
|
175
|
-
if (!match) throw new Error('SAPISID cookie missing — not signed in to Google.');
|
|
176
|
-
return createHash('sha256').update(match[1]).digest('hex').slice(0, 16);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
async function bootstrapSession(cookieHeader: string): Promise<WizSession> {
|
|
180
|
-
const accountFingerprint = accountFingerprintFromCookies(cookieHeader);
|
|
181
|
-
const res = await fetch('https://photos.google.com/', {
|
|
182
|
-
headers: {
|
|
183
|
-
cookie: cookieHeader,
|
|
184
|
-
'user-agent': USER_AGENT,
|
|
185
|
-
accept: 'text/html,application/xhtml+xml',
|
|
186
|
-
'accept-language': 'en-US,en;q=0.9',
|
|
187
|
-
},
|
|
188
|
-
});
|
|
189
|
-
const html = await res.text();
|
|
190
|
-
const m = html.match(/window\.WIZ_global_data\s*=\s*(\{[\s\S]*?\});/);
|
|
191
|
-
if (!m) {
|
|
192
|
-
const title = html.match(/<title>([^<]*)/)?.[1] ?? '';
|
|
193
|
-
throw new Error(
|
|
194
|
-
`Google Photos session invalid (title="${title}"). Sign in at photos.google.com in Chrome first.`
|
|
195
|
-
);
|
|
196
|
-
}
|
|
197
|
-
const wiz = JSON.parse(m[1]) as Record<string, unknown>;
|
|
198
|
-
const sid = wiz.FdrFJe as string;
|
|
199
|
-
const bl = wiz.cfb2h as string;
|
|
200
|
-
const at = wiz.SNlM0e as string;
|
|
201
|
-
if (!sid || !bl || !at) {
|
|
202
|
-
throw new Error('WIZ tokens missing — auth flow may have changed.');
|
|
203
|
-
}
|
|
204
|
-
return { sid, bl, at, cookieHeader, accountFingerprint };
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
// ── batchexecute RPC helpers ───────────────────────────────────
|
|
208
|
-
|
|
209
|
-
function parseWrb(text: string): { rpcId: string; payload: unknown }[] {
|
|
210
|
-
const out: { rpcId: string; payload: unknown }[] = [];
|
|
211
|
-
const stripped = text.replace(/^\)\]\}'?\n+/, '');
|
|
212
|
-
const nl = stripped.indexOf('\n');
|
|
213
|
-
if (nl === -1) return out;
|
|
214
|
-
const rest = stripped.substring(nl + 1);
|
|
215
|
-
const jsonPart = rest.replace(/\n\d+\n.*$/s, '').trim();
|
|
216
|
-
try {
|
|
217
|
-
const outer = JSON.parse(jsonPart);
|
|
218
|
-
for (const e of outer) {
|
|
219
|
-
if (Array.isArray(e) && e[0] === 'wrb.fr' && typeof e[2] === 'string') {
|
|
220
|
-
try {
|
|
221
|
-
out.push({ rpcId: e[1] as string, payload: JSON.parse(e[2]) });
|
|
222
|
-
} catch {
|
|
223
|
-
/* ignore */
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
} catch {
|
|
228
|
-
/* ignore */
|
|
229
|
-
}
|
|
230
|
-
return out;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
let reqCounter = 100000;
|
|
234
|
-
|
|
235
|
-
class RpcAuthError extends Error {
|
|
236
|
-
constructor(
|
|
237
|
-
public status: number,
|
|
238
|
-
rpcId: string
|
|
239
|
-
) {
|
|
240
|
-
super(`${rpcId} auth failure HTTP ${status}`);
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
class RpcTransientError extends Error {
|
|
244
|
-
constructor(
|
|
245
|
-
public status: number,
|
|
246
|
-
rpcId: string
|
|
247
|
-
) {
|
|
248
|
-
super(`${rpcId} transient HTTP ${status}`);
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
async function rpcCallOnce<T>(
|
|
253
|
-
rpcId: string,
|
|
254
|
-
args: unknown[],
|
|
255
|
-
sourcePath: string,
|
|
256
|
-
s: WizSession
|
|
257
|
-
): Promise<T | null> {
|
|
258
|
-
const reqid = ++reqCounter;
|
|
259
|
-
const url =
|
|
260
|
-
'https://photos.google.com/_/PhotosUi/data/batchexecute' +
|
|
261
|
-
`?rpcids=${rpcId}` +
|
|
262
|
-
`&source-path=${encodeURIComponent(sourcePath)}` +
|
|
263
|
-
`&f.sid=${s.sid}&bl=${encodeURIComponent(s.bl)}` +
|
|
264
|
-
`&hl=en&soc-app=165&soc-platform=1&soc-device=1&_reqid=${reqid}&rt=c`;
|
|
265
|
-
const fReq = JSON.stringify([[[rpcId, JSON.stringify(args), null, '1']]]);
|
|
266
|
-
const body = `f.req=${encodeURIComponent(fReq)}&at=${encodeURIComponent(s.at)}&`;
|
|
267
|
-
|
|
268
|
-
const res = await fetch(url, {
|
|
269
|
-
method: 'POST',
|
|
270
|
-
headers: {
|
|
271
|
-
'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
|
|
272
|
-
cookie: s.cookieHeader,
|
|
273
|
-
origin: 'https://photos.google.com',
|
|
274
|
-
referer: 'https://photos.google.com/',
|
|
275
|
-
'x-same-domain': '1',
|
|
276
|
-
'user-agent': USER_AGENT,
|
|
277
|
-
},
|
|
278
|
-
body,
|
|
279
|
-
});
|
|
280
|
-
if (res.status === 401 || res.status === 403) throw new RpcAuthError(res.status, rpcId);
|
|
281
|
-
if (res.status === 429 || res.status >= 500) throw new RpcTransientError(res.status, rpcId);
|
|
282
|
-
if (!res.ok) throw new Error(`${rpcId} HTTP ${res.status}`);
|
|
283
|
-
const text = await res.text();
|
|
284
|
-
const match = parseWrb(text).find((e) => e.rpcId === rpcId);
|
|
285
|
-
return (match?.payload as T) ?? null;
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
interface MutableSession {
|
|
289
|
-
current: WizSession;
|
|
290
|
-
refresh: () => Promise<WizSession>;
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
async function rpcCall<T>(
|
|
294
|
-
rpcId: string,
|
|
295
|
-
args: unknown[],
|
|
296
|
-
sourcePath: string,
|
|
297
|
-
ms: MutableSession
|
|
298
|
-
): Promise<T | null> {
|
|
299
|
-
let refreshed = false;
|
|
300
|
-
for (let attempt = 0; attempt < 4; attempt++) {
|
|
301
|
-
try {
|
|
302
|
-
return await rpcCallOnce<T>(rpcId, args, sourcePath, ms.current);
|
|
303
|
-
} catch (e) {
|
|
304
|
-
if (e instanceof RpcAuthError) {
|
|
305
|
-
if (refreshed) throw e;
|
|
306
|
-
sdkLogger.warn(
|
|
307
|
-
{ rpcId, status: e.status },
|
|
308
|
-
'[GooglePhotos] auth error — refreshing session'
|
|
309
|
-
);
|
|
310
|
-
ms.current = await ms.refresh();
|
|
311
|
-
refreshed = true;
|
|
312
|
-
continue;
|
|
313
|
-
}
|
|
314
|
-
if (e instanceof RpcTransientError) {
|
|
315
|
-
if (attempt === 3) throw e;
|
|
316
|
-
const delay = 500 * 2 ** attempt;
|
|
317
|
-
await new Promise((r) => setTimeout(r, delay));
|
|
318
|
-
continue;
|
|
319
|
-
}
|
|
320
|
-
throw e;
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
return null;
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// ── Timeline (lcxiM) extractor ─────────────────────────────────
|
|
327
|
-
|
|
328
|
-
function extractTimelinePhoto(row: unknown): TimelinePhoto | null {
|
|
329
|
-
if (!Array.isArray(row)) return null;
|
|
330
|
-
const mediaKey = row[0];
|
|
331
|
-
const img = row[1];
|
|
332
|
-
const dt = row[2];
|
|
333
|
-
if (typeof mediaKey !== 'string' || !Array.isArray(img) || typeof dt !== 'number') return null;
|
|
334
|
-
const cdnUrl = img[0];
|
|
335
|
-
const width = img[1];
|
|
336
|
-
const height = img[2];
|
|
337
|
-
if (typeof cdnUrl !== 'string' || typeof width !== 'number' || typeof height !== 'number') {
|
|
338
|
-
return null;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
const photo: TimelinePhoto = {
|
|
342
|
-
mediaKey,
|
|
343
|
-
cdnUrl,
|
|
344
|
-
width,
|
|
345
|
-
height,
|
|
346
|
-
dateTaken: new Date(dt),
|
|
347
|
-
...(typeof row[5] === 'number' ? { modifiedAt: new Date(row[5]) } : {}),
|
|
348
|
-
};
|
|
349
|
-
|
|
350
|
-
const extras = row[9];
|
|
351
|
-
if (extras && typeof extras === 'object' && !Array.isArray(extras)) {
|
|
352
|
-
const loc = (extras as Record<string, unknown>)['129168200'];
|
|
353
|
-
if (Array.isArray(loc) && Array.isArray(loc[1])) {
|
|
354
|
-
const locData = loc[1] as unknown[];
|
|
355
|
-
const coords = locData[0];
|
|
356
|
-
if (Array.isArray(coords) && coords.length >= 2) {
|
|
357
|
-
const lat = coords[0];
|
|
358
|
-
const lng = coords[1];
|
|
359
|
-
if (typeof lat === 'number' && typeof lng === 'number') {
|
|
360
|
-
photo.latitude = lat / 1e7;
|
|
361
|
-
photo.longitude = lng / 1e7;
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
const placeInfo = locData[4];
|
|
365
|
-
if (Array.isArray(placeInfo) && Array.isArray(placeInfo[0])) {
|
|
366
|
-
const place = placeInfo[0] as unknown[];
|
|
367
|
-
if (Array.isArray(place[1]) && Array.isArray(place[1][0])) {
|
|
368
|
-
const name = (place[1][0] as unknown[])[0];
|
|
369
|
-
if (typeof name === 'string') photo.locationName = name;
|
|
370
|
-
}
|
|
371
|
-
if (typeof place[2] === 'string') photo.placeId = place[2];
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
return photo;
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
async function paginateTimeline(
|
|
379
|
-
ms: MutableSession,
|
|
380
|
-
opts: { maxPages?: number; stopBefore?: Date }
|
|
381
|
-
): Promise<{
|
|
382
|
-
photos: TimelinePhoto[];
|
|
383
|
-
rowsSeen: number;
|
|
384
|
-
rowsRejected: number;
|
|
385
|
-
reachedCutoff: boolean;
|
|
386
|
-
hitMaxPages: boolean;
|
|
387
|
-
exhausted: boolean;
|
|
388
|
-
apiCalls: number;
|
|
389
|
-
}> {
|
|
390
|
-
const photos: TimelinePhoto[] = [];
|
|
391
|
-
let cursor: string | null = null;
|
|
392
|
-
let page = 0;
|
|
393
|
-
let apiCalls = 0;
|
|
394
|
-
let rowsSeen = 0;
|
|
395
|
-
let rowsRejected = 0;
|
|
396
|
-
let reachedCutoff = false;
|
|
397
|
-
let exhausted = false;
|
|
398
|
-
let hitMaxPages = false;
|
|
399
|
-
|
|
400
|
-
while (true) {
|
|
401
|
-
if (opts.maxPages != null && page >= opts.maxPages) {
|
|
402
|
-
hitMaxPages = true;
|
|
403
|
-
break;
|
|
404
|
-
}
|
|
405
|
-
const args: unknown[] = cursor ? [cursor] : [];
|
|
406
|
-
apiCalls++;
|
|
407
|
-
const payload = (await rpcCall('lcxiM', args, '/', ms)) as unknown[] | null;
|
|
408
|
-
if (!Array.isArray(payload)) {
|
|
409
|
-
exhausted = true;
|
|
410
|
-
break;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
const rows = Array.isArray(payload[0]) ? (payload[0] as unknown[]) : [];
|
|
414
|
-
const nextCursor = typeof payload[1] === 'string' ? (payload[1] as string) : null;
|
|
415
|
-
|
|
416
|
-
let hitStop = false;
|
|
417
|
-
for (const r of rows) {
|
|
418
|
-
rowsSeen++;
|
|
419
|
-
const p = extractTimelinePhoto(r);
|
|
420
|
-
if (!p) {
|
|
421
|
-
rowsRejected++;
|
|
422
|
-
continue;
|
|
423
|
-
}
|
|
424
|
-
if (opts.stopBefore && p.dateTaken <= opts.stopBefore) {
|
|
425
|
-
hitStop = true;
|
|
426
|
-
reachedCutoff = true;
|
|
427
|
-
break;
|
|
428
|
-
}
|
|
429
|
-
photos.push(p);
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
if ((page + 1) % 10 === 0) {
|
|
433
|
-
sdkLogger.info({ page: page + 1, photos: photos.length }, '[GooglePhotos] timeline progress');
|
|
434
|
-
}
|
|
435
|
-
page++;
|
|
436
|
-
if (hitStop) break;
|
|
437
|
-
if (!nextCursor) {
|
|
438
|
-
exhausted = true;
|
|
439
|
-
break;
|
|
440
|
-
}
|
|
441
|
-
cursor = nextCursor;
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
// A high reject rate means Google changed the row shape — loud signal so we
|
|
445
|
-
// catch it before downstream consumers see a quiet drop in ingestion.
|
|
446
|
-
if (rowsSeen > 100 && rowsRejected / rowsSeen > 0.2) {
|
|
447
|
-
sdkLogger.warn(
|
|
448
|
-
{ rowsSeen, rowsRejected, rate: rowsRejected / rowsSeen },
|
|
449
|
-
'[GooglePhotos] high extraction reject rate — lcxiM row shape may have changed'
|
|
450
|
-
);
|
|
451
|
-
}
|
|
452
|
-
return { photos, rowsSeen, rowsRejected, reachedCutoff, hitMaxPages, exhausted, apiCalls };
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
// ── fDcn4b EXIF extractor ──────────────────────────────────────
|
|
456
|
-
|
|
457
|
-
function extractDetailFromFDcn4b(payload: unknown): PhotoDetail | null {
|
|
458
|
-
if (!Array.isArray(payload)) return null;
|
|
459
|
-
const root = Array.isArray(payload[0]) ? (payload[0] as unknown[]) : (payload as unknown[]);
|
|
460
|
-
if (!Array.isArray(root)) return null;
|
|
461
|
-
|
|
462
|
-
const get = (i: number) => root[i];
|
|
463
|
-
const out: PhotoDetail = {};
|
|
464
|
-
if (typeof get(2) === 'string' && (get(2) as string).length > 0) out.filename = get(2) as string;
|
|
465
|
-
if (typeof get(4) === 'number') out.tzOffsetMin = Math.round((get(4) as number) / 60000);
|
|
466
|
-
|
|
467
|
-
const exif = get(23);
|
|
468
|
-
if (Array.isArray(exif)) {
|
|
469
|
-
if (typeof exif[0] === 'string') out.cameraMake = exif[0] as string;
|
|
470
|
-
if (typeof exif[1] === 'string') out.cameraModel = exif[1] as string;
|
|
471
|
-
if (typeof exif[3] === 'number') out.focalOrStop = exif[3] as number;
|
|
472
|
-
if (typeof exif[4] === 'number') out.aperture = exif[4] as number;
|
|
473
|
-
if (typeof exif[5] === 'number') out.iso = exif[5] as number;
|
|
474
|
-
if (typeof exif[6] === 'number') out.shutterSec = exif[6] as number;
|
|
475
|
-
}
|
|
476
|
-
return Object.keys(out).length ? out : null;
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
async function fillExif(
|
|
480
|
-
photos: TimelinePhoto[],
|
|
481
|
-
ms: MutableSession,
|
|
482
|
-
concurrency: number
|
|
483
|
-
): Promise<{ results: Map<string, PhotoDetail>; transientFailures: number }> {
|
|
484
|
-
const results = new Map<string, PhotoDetail>();
|
|
485
|
-
let cursor = 0;
|
|
486
|
-
let transientFailures = 0;
|
|
487
|
-
let fatal: Error | null = null;
|
|
488
|
-
|
|
489
|
-
async function worker() {
|
|
490
|
-
while (cursor < photos.length && !fatal) {
|
|
491
|
-
const idx = cursor++;
|
|
492
|
-
const p = photos[idx];
|
|
493
|
-
try {
|
|
494
|
-
const payload = await rpcCall('fDcn4b', [p.mediaKey], `/photo/${p.mediaKey}`, ms);
|
|
495
|
-
const d = extractDetailFromFDcn4b(payload);
|
|
496
|
-
if (d) results.set(p.mediaKey, d);
|
|
497
|
-
} catch (e) {
|
|
498
|
-
if (e instanceof RpcAuthError) {
|
|
499
|
-
fatal = e;
|
|
500
|
-
return;
|
|
501
|
-
}
|
|
502
|
-
transientFailures++;
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
|
|
507
|
-
if (fatal) throw fatal;
|
|
508
|
-
return { results, transientFailures };
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
// ── Event envelope ─────────────────────────────────────────────
|
|
512
|
-
|
|
513
|
-
const VIDEO_EXT = /\.(mov|mp4|m4v|avi|mkv|webm|3gp)$/i;
|
|
514
|
-
|
|
515
|
-
function isVideo(exif?: PhotoDetail): boolean {
|
|
516
|
-
return !!(exif?.filename && VIDEO_EXT.test(exif.filename));
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
function photoToEvent(p: TimelinePhoto, exif?: PhotoDetail): EventEnvelope {
|
|
520
|
-
const cameraMake = exif?.cameraMake;
|
|
521
|
-
const cameraModel = exif?.cameraModel;
|
|
522
|
-
const video = isVideo(exif);
|
|
523
|
-
return {
|
|
524
|
-
origin_id: `gp_${p.mediaKey}`,
|
|
525
|
-
title: exif?.filename || `Photo ${p.dateTaken.toISOString().split('T')[0]}`,
|
|
526
|
-
payload_text: '',
|
|
527
|
-
source_url: `https://photos.google.com/photo/${p.mediaKey}`,
|
|
528
|
-
occurred_at: p.dateTaken,
|
|
529
|
-
origin_type: video ? 'video' : 'photo',
|
|
530
|
-
score: 0,
|
|
531
|
-
metadata: {
|
|
532
|
-
media_key: p.mediaKey,
|
|
533
|
-
date_taken: p.dateTaken.toISOString(),
|
|
534
|
-
cdn_url: p.cdnUrl,
|
|
535
|
-
thumbnail_url: `${p.cdnUrl}=w400-h400`,
|
|
536
|
-
width: p.width,
|
|
537
|
-
height: p.height,
|
|
538
|
-
is_video: video,
|
|
539
|
-
...(p.modifiedAt ? { modified_at: p.modifiedAt.toISOString() } : {}),
|
|
540
|
-
...(p.latitude != null ? { latitude: p.latitude } : {}),
|
|
541
|
-
...(p.longitude != null ? { longitude: p.longitude } : {}),
|
|
542
|
-
...(p.locationName ? { location_name: p.locationName } : {}),
|
|
543
|
-
...(p.placeId ? { place_id: p.placeId } : {}),
|
|
544
|
-
...(exif?.filename ? { filename: exif.filename } : {}),
|
|
545
|
-
...(cameraMake ? { camera_make: cameraMake } : {}),
|
|
546
|
-
...(cameraModel ? { camera_model: cameraModel } : {}),
|
|
547
|
-
...(exif?.aperture != null ? { aperture: exif.aperture } : {}),
|
|
548
|
-
...(exif?.iso != null ? { iso: exif.iso } : {}),
|
|
549
|
-
...(exif?.shutterSec != null ? { shutter_sec: exif.shutterSec } : {}),
|
|
550
|
-
...(exif?.focalOrStop != null ? { focal_or_stop: exif.focalOrStop } : {}),
|
|
551
|
-
...(exif?.tzOffsetMin != null ? { tz_offset_min: exif.tzOffsetMin } : {}),
|
|
552
|
-
},
|
|
553
|
-
};
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
// ── Connector ──────────────────────────────────────────────────
|
|
557
|
-
|
|
558
|
-
export default class GooglePhotosConnector extends ConnectorRuntime {
|
|
559
|
-
readonly definition: ConnectorDefinition = {
|
|
560
|
-
key: 'google_photos',
|
|
561
|
-
name: 'Google Photos',
|
|
562
|
-
description:
|
|
563
|
-
'Ingests Google Photos metadata via direct batchexecute RPCs using cookies captured from a running Chrome session.',
|
|
564
|
-
version: '6.1.0',
|
|
565
|
-
faviconDomain: 'photos.google.com',
|
|
566
|
-
authSchema: {
|
|
567
|
-
methods: [
|
|
568
|
-
{
|
|
569
|
-
type: 'browser',
|
|
570
|
-
capture: 'cdp',
|
|
571
|
-
description:
|
|
572
|
-
'Grabs session cookies from your running Chrome via Chrome DevTools Protocol. Chrome must be launched with --remote-debugging-port=9222 and signed in to Google Photos.',
|
|
573
|
-
defaultCdpUrl: 'auto',
|
|
574
|
-
},
|
|
575
|
-
],
|
|
576
|
-
},
|
|
577
|
-
feeds: {
|
|
578
|
-
photos: {
|
|
579
|
-
key: 'photos',
|
|
580
|
-
name: 'Photos',
|
|
581
|
-
description: 'Sync photo metadata from your Google Photos library.',
|
|
582
|
-
configSchema: {
|
|
583
|
-
type: 'object',
|
|
584
|
-
properties: {
|
|
585
|
-
max_pages: {
|
|
586
|
-
type: 'integer',
|
|
587
|
-
minimum: 1,
|
|
588
|
-
description: 'Cap on lcxiM pages (≈300 photos each). Omit for full backfill.',
|
|
589
|
-
},
|
|
590
|
-
exif_limit: {
|
|
591
|
-
type: 'integer',
|
|
592
|
-
minimum: 0,
|
|
593
|
-
default: 1000,
|
|
594
|
-
description: 'Max photos to enrich with EXIF per run. Set 0 to skip EXIF.',
|
|
595
|
-
},
|
|
596
|
-
exif_concurrency: {
|
|
597
|
-
type: 'integer',
|
|
598
|
-
minimum: 1,
|
|
599
|
-
maximum: 20,
|
|
600
|
-
default: 10,
|
|
601
|
-
},
|
|
602
|
-
},
|
|
603
|
-
},
|
|
604
|
-
eventKinds: {
|
|
605
|
-
photo: {
|
|
606
|
-
description: 'A photo with metadata (date, dimensions, GPS, place, EXIF)',
|
|
607
|
-
metadataSchema: {
|
|
608
|
-
type: 'object',
|
|
609
|
-
properties: {
|
|
610
|
-
media_key: { type: 'string' },
|
|
611
|
-
date_taken: { type: 'string' },
|
|
612
|
-
modified_at: { type: 'string' },
|
|
613
|
-
cdn_url: { type: 'string' },
|
|
614
|
-
thumbnail_url: { type: 'string' },
|
|
615
|
-
width: { type: 'number' },
|
|
616
|
-
height: { type: 'number' },
|
|
617
|
-
is_video: { type: 'boolean' },
|
|
618
|
-
latitude: { type: 'number' },
|
|
619
|
-
longitude: { type: 'number' },
|
|
620
|
-
location_name: { type: 'string' },
|
|
621
|
-
place_id: { type: 'string' },
|
|
622
|
-
filename: { type: 'string' },
|
|
623
|
-
camera_make: { type: 'string' },
|
|
624
|
-
camera_model: { type: 'string' },
|
|
625
|
-
aperture: { type: 'number' },
|
|
626
|
-
iso: { type: 'number' },
|
|
627
|
-
shutter_sec: { type: 'number' },
|
|
628
|
-
focal_or_stop: { type: 'number' },
|
|
629
|
-
tz_offset_min: { type: 'number' },
|
|
630
|
-
},
|
|
631
|
-
},
|
|
632
|
-
},
|
|
633
|
-
video: {
|
|
634
|
-
description: 'A video with metadata (date, dimensions, GPS, place)',
|
|
635
|
-
metadataSchema: {
|
|
636
|
-
type: 'object',
|
|
637
|
-
properties: {
|
|
638
|
-
media_key: { type: 'string' },
|
|
639
|
-
date_taken: { type: 'string' },
|
|
640
|
-
modified_at: { type: 'string' },
|
|
641
|
-
cdn_url: { type: 'string' },
|
|
642
|
-
thumbnail_url: { type: 'string' },
|
|
643
|
-
width: { type: 'number' },
|
|
644
|
-
height: { type: 'number' },
|
|
645
|
-
is_video: { type: 'boolean' },
|
|
646
|
-
latitude: { type: 'number' },
|
|
647
|
-
longitude: { type: 'number' },
|
|
648
|
-
location_name: { type: 'string' },
|
|
649
|
-
place_id: { type: 'string' },
|
|
650
|
-
filename: { type: 'string' },
|
|
651
|
-
},
|
|
652
|
-
},
|
|
653
|
-
},
|
|
654
|
-
},
|
|
655
|
-
},
|
|
656
|
-
},
|
|
657
|
-
};
|
|
658
|
-
|
|
659
|
-
async sync(ctx: SyncContext): Promise<SyncResult> {
|
|
660
|
-
const config = ctx.config as Record<string, unknown>;
|
|
661
|
-
const checkpoint = (ctx.checkpoint ?? {}) as GooglePhotosCheckpoint;
|
|
662
|
-
|
|
663
|
-
const cdpUrl = await resolveCdpUrl((ctx.sessionState as any)?.cdp_url || 'auto', {
|
|
664
|
-
loggerLabel: 'GooglePhotos',
|
|
665
|
-
});
|
|
666
|
-
|
|
667
|
-
const bootstrap = async (): Promise<WizSession> => {
|
|
668
|
-
const cookieHeader = await fetchCookiesViaCdp(cdpUrl);
|
|
669
|
-
return bootstrapSession(cookieHeader);
|
|
670
|
-
};
|
|
671
|
-
|
|
672
|
-
const session = await bootstrap();
|
|
673
|
-
sdkLogger.info(
|
|
674
|
-
{ accountFingerprint: session.accountFingerprint },
|
|
675
|
-
'[GooglePhotos] session bootstrapped'
|
|
676
|
-
);
|
|
677
|
-
|
|
678
|
-
if (
|
|
679
|
-
checkpoint.account_fingerprint &&
|
|
680
|
-
checkpoint.account_fingerprint !== session.accountFingerprint
|
|
681
|
-
) {
|
|
682
|
-
throw new Error(
|
|
683
|
-
`Google account mismatch — checkpoint belongs to ${checkpoint.account_fingerprint}, ` +
|
|
684
|
-
`current Chrome session is ${session.accountFingerprint}. ` +
|
|
685
|
-
'Switch Chrome to the original account or create a new connection.'
|
|
686
|
-
);
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
const ms: MutableSession = { current: session, refresh: bootstrap };
|
|
690
|
-
|
|
691
|
-
const stopBefore = checkpoint.last_timestamp ? new Date(checkpoint.last_timestamp) : undefined;
|
|
692
|
-
const maxPages = typeof config.max_pages === 'number' ? config.max_pages : undefined;
|
|
693
|
-
|
|
694
|
-
const started = Date.now();
|
|
695
|
-
const timeline = await paginateTimeline(ms, {
|
|
696
|
-
...(maxPages != null ? { maxPages } : {}),
|
|
697
|
-
...(stopBefore ? { stopBefore } : {}),
|
|
698
|
-
});
|
|
699
|
-
sdkLogger.info(
|
|
700
|
-
{
|
|
701
|
-
photos: timeline.photos.length,
|
|
702
|
-
pages: timeline.apiCalls,
|
|
703
|
-
exhausted: timeline.exhausted,
|
|
704
|
-
hitMaxPages: timeline.hitMaxPages,
|
|
705
|
-
reachedCutoff: timeline.reachedCutoff,
|
|
706
|
-
ms: Date.now() - started,
|
|
707
|
-
},
|
|
708
|
-
'[GooglePhotos] timeline done'
|
|
709
|
-
);
|
|
710
|
-
|
|
711
|
-
const exifLimit = typeof config.exif_limit === 'number' ? config.exif_limit : 1000;
|
|
712
|
-
const exifConcurrency =
|
|
713
|
-
typeof config.exif_concurrency === 'number' ? config.exif_concurrency : 10;
|
|
714
|
-
|
|
715
|
-
let exifMap: Map<string, PhotoDetail> = new Map();
|
|
716
|
-
let exifFailures = 0;
|
|
717
|
-
if (exifLimit > 0 && timeline.photos.length > 0) {
|
|
718
|
-
const targets = timeline.photos.slice(0, exifLimit);
|
|
719
|
-
const exifStart = Date.now();
|
|
720
|
-
const exif = await fillExif(targets, ms, exifConcurrency);
|
|
721
|
-
exifMap = exif.results;
|
|
722
|
-
exifFailures = exif.transientFailures;
|
|
723
|
-
sdkLogger.info(
|
|
724
|
-
{
|
|
725
|
-
enriched: exifMap.size,
|
|
726
|
-
target: targets.length,
|
|
727
|
-
transientFailures: exifFailures,
|
|
728
|
-
ms: Date.now() - exifStart,
|
|
729
|
-
},
|
|
730
|
-
'[GooglePhotos] exif done'
|
|
731
|
-
);
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
const events = timeline.photos.map((p) => photoToEvent(p, exifMap.get(p.mediaKey)));
|
|
735
|
-
|
|
736
|
-
// Only advance checkpoint when we know we've seen everything newer than the
|
|
737
|
-
// prior checkpoint. hitMaxPages with no cutoff means there are older photos
|
|
738
|
-
// we didn't fetch — advancing would permanently skip them on the next run.
|
|
739
|
-
const canAdvance = timeline.reachedCutoff || timeline.exhausted;
|
|
740
|
-
const newest = timeline.photos[0];
|
|
741
|
-
const newCheckpoint: GooglePhotosCheckpoint =
|
|
742
|
-
canAdvance && newest
|
|
743
|
-
? {
|
|
744
|
-
last_photo_id: newest.mediaKey,
|
|
745
|
-
last_timestamp: newest.dateTaken.toISOString(),
|
|
746
|
-
account_fingerprint: session.accountFingerprint,
|
|
747
|
-
}
|
|
748
|
-
: { ...checkpoint, account_fingerprint: session.accountFingerprint };
|
|
749
|
-
|
|
750
|
-
const videoCount = events.filter((e) => e.origin_type === 'video').length;
|
|
751
|
-
|
|
752
|
-
return {
|
|
753
|
-
events,
|
|
754
|
-
checkpoint: newCheckpoint as unknown as Record<string, unknown>,
|
|
755
|
-
auth_update: {},
|
|
756
|
-
metadata: {
|
|
757
|
-
items_found: events.length,
|
|
758
|
-
videos: videoCount,
|
|
759
|
-
photos: events.length - videoCount,
|
|
760
|
-
timeline_pages: timeline.apiCalls,
|
|
761
|
-
rows_seen: timeline.rowsSeen,
|
|
762
|
-
rows_rejected: timeline.rowsRejected,
|
|
763
|
-
exif_fetched: exifMap.size,
|
|
764
|
-
exif_transient_failures: exifFailures,
|
|
765
|
-
reached_cutoff: timeline.reachedCutoff,
|
|
766
|
-
exhausted: timeline.exhausted,
|
|
767
|
-
hit_max_pages: timeline.hitMaxPages,
|
|
768
|
-
checkpoint_advanced: canAdvance,
|
|
769
|
-
},
|
|
770
|
-
};
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
async execute(_ctx: ActionContext): Promise<ActionResult> {
|
|
774
|
-
return { success: false, error: 'Actions not supported' };
|
|
775
|
-
}
|
|
776
|
-
}
|