@lobu/cli 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/_lib/apply/apply-cmd.d.ts.map +1 -1
- package/dist/commands/_lib/apply/apply-cmd.js +160 -12
- package/dist/commands/_lib/apply/apply-cmd.js.map +1 -1
- package/dist/commands/_lib/apply/client.d.ts +106 -0
- package/dist/commands/_lib/apply/client.d.ts.map +1 -1
- package/dist/commands/_lib/apply/client.js +163 -2
- package/dist/commands/_lib/apply/client.js.map +1 -1
- package/dist/commands/_lib/apply/desired-state.d.ts +53 -0
- package/dist/commands/_lib/apply/desired-state.d.ts.map +1 -1
- package/dist/commands/_lib/apply/desired-state.js +182 -5
- package/dist/commands/_lib/apply/desired-state.js.map +1 -1
- package/dist/commands/_lib/apply/diff.d.ts +12 -1
- package/dist/commands/_lib/apply/diff.d.ts.map +1 -1
- package/dist/commands/_lib/apply/diff.js +106 -7
- package/dist/commands/_lib/apply/diff.js.map +1 -1
- package/dist/commands/_lib/connector-loader.d.ts +3 -0
- package/dist/commands/_lib/connector-loader.d.ts.map +1 -0
- package/dist/commands/_lib/connector-loader.js +129 -0
- package/dist/commands/_lib/connector-loader.js.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts +35 -0
- package/dist/commands/_lib/connector-run-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/connector-run-cmd.js +351 -0
- package/dist/commands/_lib/connector-run-cmd.js.map +1 -0
- package/dist/commands/_lib/export/export-cmd.d.ts +35 -0
- package/dist/commands/_lib/export/export-cmd.d.ts.map +1 -0
- package/dist/commands/_lib/export/export-cmd.js +329 -0
- package/dist/commands/_lib/export/export-cmd.js.map +1 -0
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +11 -14
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/chat.d.ts.map +1 -1
- package/dist/commands/chat.js +19 -5
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/connector.d.ts +3 -0
- package/dist/commands/connector.d.ts.map +1 -0
- package/dist/commands/connector.js +5 -0
- package/dist/commands/connector.js.map +1 -0
- package/dist/commands/context.d.ts +7 -0
- package/dist/commands/context.d.ts.map +1 -1
- package/dist/commands/context.js +19 -2
- package/dist/commands/context.js.map +1 -1
- package/dist/commands/dev.d.ts +15 -0
- package/dist/commands/dev.d.ts.map +1 -1
- package/dist/commands/dev.js +156 -4
- package/dist/commands/dev.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +2 -3
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +12 -13
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +5 -1
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/login.d.ts.map +1 -1
- package/dist/commands/login.js +22 -16
- package/dist/commands/login.js.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.d.ts.map +1 -1
- package/dist/commands/memory/_lib/browser-auth-cmd.js +15 -144
- package/dist/commands/memory/_lib/browser-auth-cmd.js.map +1 -1
- package/dist/commands/token.d.ts.map +1 -1
- package/dist/commands/token.js +1 -4
- package/dist/commands/token.js.map +1 -1
- package/dist/commands/validate.d.ts.map +1 -1
- package/dist/commands/validate.js +4 -13
- package/dist/commands/validate.js.map +1 -1
- package/dist/config/loader.js +2 -2
- package/dist/config/loader.js.map +1 -1
- package/dist/connectors/README.md +0 -1
- package/dist/connectors/apple_photos.ts +178 -0
- package/dist/connectors/browser-scraper-utils.ts +76 -0
- package/dist/connectors/chrome.ts +351 -0
- package/dist/connectors/chrome_bookmarks.ts +79 -0
- package/dist/connectors/chrome_downloads.ts +80 -0
- package/dist/connectors/chrome_history.ts +80 -0
- package/dist/connectors/github.ts +1 -0
- package/dist/connectors/google_calendar.ts +14 -2
- package/dist/connectors/google_play.ts +22 -2
- package/dist/connectors/hackernews.ts +37 -2
- package/dist/connectors/index.ts +15 -1
- package/dist/connectors/reddit.ts +1 -0
- package/dist/connectors/revolut.ts +10 -13
- package/dist/connectors/rss.ts +33 -8
- package/dist/connectors/trustpilot.ts +31 -20
- package/dist/connectors/website.ts +7 -68
- package/dist/connectors/whatsapp.ts +12 -21
- package/dist/db/migrations/20260514130000_connection_action_modes.sql +103 -0
- package/dist/db/migrations/20260514160000_auth_profiles_mirror_mode.sql +32 -0
- package/dist/db/migrations/20260515120000_agents_per_org_pk.sql +66 -0
- package/dist/db/migrations/20260515150000_geo_enrichment.sql +208 -0
- package/dist/db/migrations/20260515160000_drop_agents_org_id_unique.sql +24 -0
- package/dist/db/migrations/20260515170000_auth_profiles_default_for_connector.sql +23 -0
- package/dist/db/migrations/20260516120000_agents_per_org_pk_swap.sql +125 -0
- package/dist/db/migrations/20260516200000_events_search_tsv.sql +134 -0
- package/dist/db/migrations/20260516200100_events_lifecycle_changes_index.sql +25 -0
- package/dist/db/migrations/20260517010000_drop_unused_indexes.sql +49 -0
- package/dist/db/migrations/20260517020000_softdelete_orphan_feeds.sql +56 -0
- package/dist/db/migrations/20260517030000_pat_worker_id_binding.sql +27 -0
- package/dist/db/migrations/20260517040000_archive_orphan_watchers.sql +30 -0
- package/dist/db/migrations/20260517050000_watcher_agent_id_not_null.sql +34 -0
- package/dist/db/migrations/20260517060000_watcher_schema_additions.sql +78 -0
- package/dist/db/migrations/20260517150000_goals_primitive.sql +55 -0
- package/dist/db/migrations/20260517160000_drop_goals_primitive.sql +45 -0
- package/dist/db/migrations/20260518000000_pending_interactions.sql +49 -0
- package/dist/db/migrations/20260518010000_runs_heartbeat_reaper_index.sql +22 -0
- package/dist/db/migrations/20260518020000_runs_heartbeat_inflight_narrow.sql +36 -0
- package/dist/db/migrations/20260518040000_agent_transcript_snapshot.sql +54 -0
- package/dist/db/migrations/20260518050000_runs_denormalize_agent_conversation.sql +36 -0
- package/dist/db/migrations/20260518060000_revert_runs_denormalize.sql +29 -0
- package/dist/db/migrations/20260518070000_runs_heartbeat_inflight_widen.sql +33 -0
- package/dist/eval/client.d.ts.map +1 -1
- package/dist/eval/client.js +11 -0
- package/dist/eval/client.js.map +1 -1
- package/dist/eval/grader.js +2 -1
- package/dist/eval/grader.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +84 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/context.d.ts +13 -1
- package/dist/internal/context.d.ts.map +1 -1
- package/dist/internal/context.js +83 -8
- package/dist/internal/context.js.map +1 -1
- package/dist/internal/credentials.d.ts +5 -0
- package/dist/internal/credentials.d.ts.map +1 -1
- package/dist/internal/credentials.js +75 -1
- package/dist/internal/credentials.js.map +1 -1
- package/dist/internal/index.d.ts +2 -2
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +2 -2
- package/dist/internal/index.js.map +1 -1
- package/dist/internal/local-env.d.ts.map +1 -1
- package/dist/internal/local-env.js +9 -2
- package/dist/internal/local-env.js.map +1 -1
- package/dist/server.bundle.mjs +7085 -2832
- package/dist/start-local.bundle.mjs +8269 -3656
- package/package.json +7 -5
- package/dist/connectors/google_photos.ts +0 -776
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type SyncResult,
|
|
21
21
|
} from '@lobu/connector-sdk';
|
|
22
22
|
import type { Page } from 'playwright';
|
|
23
|
+
import { validatePublicUrl } from './browser-scraper-utils.ts';
|
|
23
24
|
|
|
24
25
|
interface PageSection {
|
|
25
26
|
heading: string;
|
|
@@ -50,72 +51,6 @@ function shouldSkipCookieBannerText(text: string): boolean {
|
|
|
50
51
|
return countPatternMatches(normalized, COOKIE_BANNER_PATTERNS) >= 3;
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
/**
|
|
54
|
-
* Validates a URL is safe for server-side fetching.
|
|
55
|
-
* Blocks private/internal network addresses to prevent SSRF attacks.
|
|
56
|
-
*/
|
|
57
|
-
function validatePublicUrl(url: string): void {
|
|
58
|
-
let parsed: URL;
|
|
59
|
-
try {
|
|
60
|
-
parsed = new URL(url);
|
|
61
|
-
} catch {
|
|
62
|
-
throw new Error(`Invalid URL: ${url}`);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
|
|
66
|
-
throw new Error(`URL must use http: or https: protocol, got ${parsed.protocol}`);
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
const hostname = parsed.hostname.toLowerCase();
|
|
70
|
-
|
|
71
|
-
// Block localhost variants
|
|
72
|
-
if (hostname === 'localhost' || hostname === '[::1]' || hostname.endsWith('.localhost')) {
|
|
73
|
-
throw new Error(`URL must not point to localhost: ${hostname}`);
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// Block private/internal IP ranges
|
|
77
|
-
// IPv4 patterns: 127.x.x.x, 10.x.x.x, 192.168.x.x, 172.16-31.x.x, 169.254.x.x, 0.x.x.x
|
|
78
|
-
const ipv4Match = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
79
|
-
if (ipv4Match) {
|
|
80
|
-
const [, a, b] = ipv4Match.map(Number);
|
|
81
|
-
if (
|
|
82
|
-
a === 127 || // 127.0.0.0/8 loopback
|
|
83
|
-
a === 10 || // 10.0.0.0/8 private
|
|
84
|
-
(a === 172 && b >= 16 && b <= 31) || // 172.16.0.0/12 private
|
|
85
|
-
(a === 192 && b === 168) || // 192.168.0.0/16 private
|
|
86
|
-
(a === 169 && b === 254) || // 169.254.0.0/16 link-local
|
|
87
|
-
a === 0 // 0.0.0.0/8
|
|
88
|
-
) {
|
|
89
|
-
throw new Error(`URL must not point to a private/internal IP address: ${hostname}`);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Block IPv6 private ranges (bracketed notation in URLs)
|
|
94
|
-
if (hostname.startsWith('[')) {
|
|
95
|
-
const ipv6 = hostname.slice(1, -1).toLowerCase();
|
|
96
|
-
if (
|
|
97
|
-
ipv6 === '::1' ||
|
|
98
|
-
ipv6.startsWith('fe80:') || // link-local
|
|
99
|
-
ipv6.startsWith('fc') || // unique local (fc00::/7)
|
|
100
|
-
ipv6.startsWith('fd') || // unique local (fc00::/7)
|
|
101
|
-
ipv6 === '::' || // unspecified
|
|
102
|
-
ipv6.startsWith('::ffff:') // IPv4-mapped IPv6
|
|
103
|
-
) {
|
|
104
|
-
throw new Error(`URL must not point to a private/internal IPv6 address: ${hostname}`);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// Block common internal hostnames
|
|
109
|
-
if (
|
|
110
|
-
hostname.endsWith('.internal') ||
|
|
111
|
-
hostname.endsWith('.local') ||
|
|
112
|
-
hostname.endsWith('.corp') ||
|
|
113
|
-
hostname.endsWith('.lan')
|
|
114
|
-
) {
|
|
115
|
-
throw new Error(`URL must not point to an internal hostname: ${hostname}`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
54
|
export default class WebsiteConnector extends ConnectorRuntime {
|
|
120
55
|
readonly definition: ConnectorDefinition = {
|
|
121
56
|
key: 'website',
|
|
@@ -457,7 +392,11 @@ export default class WebsiteConnector extends ConnectorRuntime {
|
|
|
457
392
|
return result.join('\n');
|
|
458
393
|
}
|
|
459
394
|
|
|
460
|
-
private async fetchSitemap(sitemapUrl: string): Promise<string[]> {
|
|
395
|
+
private async fetchSitemap(sitemapUrl: string, depth = 0): Promise<string[]> {
|
|
396
|
+
// Sitemap-index recursion bound — caps fan-out from a remote sitemap that
|
|
397
|
+
// links to a sitemap that links to a sitemap... untrusted XML must not
|
|
398
|
+
// drive unbounded outbound traffic.
|
|
399
|
+
if (depth > 2) return [];
|
|
461
400
|
const response = await fetch(sitemapUrl, {
|
|
462
401
|
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; LobuBot/1.0)' },
|
|
463
402
|
});
|
|
@@ -493,7 +432,7 @@ export default class WebsiteConnector extends ConnectorRuntime {
|
|
|
493
432
|
}
|
|
494
433
|
for (const childUrl of childSitemaps.slice(0, 5)) {
|
|
495
434
|
validatePublicUrl(childUrl);
|
|
496
|
-
const childUrls = await this.fetchSitemap(childUrl);
|
|
435
|
+
const childUrls = await this.fetchSitemap(childUrl, depth + 1);
|
|
497
436
|
urls.push(...childUrls);
|
|
498
437
|
}
|
|
499
438
|
}
|
|
@@ -424,11 +424,7 @@ export default class WhatsAppConnector extends ConnectorRuntime {
|
|
|
424
424
|
},
|
|
425
425
|
};
|
|
426
426
|
} catch (error) {
|
|
427
|
-
|
|
428
|
-
sock.end(undefined);
|
|
429
|
-
} catch {
|
|
430
|
-
/* ignore */
|
|
431
|
-
}
|
|
427
|
+
safeEnd(sock);
|
|
432
428
|
throw error;
|
|
433
429
|
}
|
|
434
430
|
}
|
|
@@ -478,11 +474,7 @@ async function attemptPairing(
|
|
|
478
474
|
sock.ev.off('connection.update', handler);
|
|
479
475
|
sock.ev.off('creds.update', credsListener);
|
|
480
476
|
ctx.signal.removeEventListener('abort', onAbort);
|
|
481
|
-
|
|
482
|
-
sock.end(undefined);
|
|
483
|
-
} catch {
|
|
484
|
-
/* ignore */
|
|
485
|
-
}
|
|
477
|
+
safeEnd(sock);
|
|
486
478
|
resolve(outcome);
|
|
487
479
|
};
|
|
488
480
|
|
|
@@ -592,11 +584,7 @@ async function drainHistory(
|
|
|
592
584
|
sock.ev.off('chats.upsert', chatsListener);
|
|
593
585
|
sock.ev.off('messaging-history.set', historyListener);
|
|
594
586
|
sock.ev.off('messages.upsert', messagesListener);
|
|
595
|
-
|
|
596
|
-
sock.end(undefined);
|
|
597
|
-
} catch {
|
|
598
|
-
/* ignore */
|
|
599
|
-
}
|
|
587
|
+
safeEnd(sock);
|
|
600
588
|
};
|
|
601
589
|
|
|
602
590
|
try {
|
|
@@ -829,6 +817,14 @@ function delay(ms: number): Promise<void> {
|
|
|
829
817
|
return new Promise((r) => setTimeout(r, ms));
|
|
830
818
|
}
|
|
831
819
|
|
|
820
|
+
function safeEnd(sock: ReturnType<typeof makeWASocket>): void {
|
|
821
|
+
try {
|
|
822
|
+
sock.end(undefined);
|
|
823
|
+
} catch {
|
|
824
|
+
/* ignore */
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
|
|
832
828
|
function waitForOpen(sock: ReturnType<typeof makeWASocket>, timeoutMs: number): Promise<boolean> {
|
|
833
829
|
return new Promise((resolve) => {
|
|
834
830
|
let newLogin = false;
|
|
@@ -963,12 +959,7 @@ export function toEvent(
|
|
|
963
959
|
const text = extractText(m.message);
|
|
964
960
|
if (!text) return null;
|
|
965
961
|
|
|
966
|
-
const tsRaw =
|
|
967
|
-
typeof m.messageTimestamp === 'number'
|
|
968
|
-
? m.messageTimestamp
|
|
969
|
-
: ((m.messageTimestamp as { low?: number; toNumber?: () => number } | null)?.toNumber?.() ??
|
|
970
|
-
(m.messageTimestamp as { low?: number } | null)?.low ??
|
|
971
|
-
0);
|
|
962
|
+
const tsRaw = extractTs(m);
|
|
972
963
|
if (!tsRaw) return null;
|
|
973
964
|
const occurredAt = new Date(tsRaw * 1000);
|
|
974
965
|
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
|
|
3
|
+
-- Collapse `connection.config.auto_approve_actions` (string[]) and
|
|
4
|
+
-- `connection.config.require_approval_actions` (string[]) into a single
|
|
5
|
+
-- `action_modes` (Record<string, 'disabled' | 'approval' | 'auto'>) map.
|
|
6
|
+
--
|
|
7
|
+
-- The old two-array model couldn't express "agent must not call this op
|
|
8
|
+
-- at all" — every action the connector defined was always reachable, the
|
|
9
|
+
-- arrays only flipped approval prompts. The new map adds 'disabled' as the
|
|
10
|
+
-- third state and gives every op an explicit user-chosen mode.
|
|
11
|
+
--
|
|
12
|
+
-- Backfill rule, per row, for every op listed in either array:
|
|
13
|
+
-- op in auto_approve_actions → action_modes[op] = 'auto'
|
|
14
|
+
-- op in require_approval_actions → action_modes[op] = 'approval'
|
|
15
|
+
-- When an op appears in both, 'approval' wins (it's the stricter signal:
|
|
16
|
+
-- the user explicitly opted in to seeing an approval prompt).
|
|
17
|
+
--
|
|
18
|
+
-- Ops the user never touched are not stored in action_modes; the server
|
|
19
|
+
-- falls back to the connector's per-op `requires_approval` default at read
|
|
20
|
+
-- time, which preserves today's "all on" behavior.
|
|
21
|
+
--
|
|
22
|
+
-- We drop the two old keys in the same statement so the new state is the
|
|
23
|
+
-- only state on disk after migration.
|
|
24
|
+
|
|
25
|
+
UPDATE public.connections
|
|
26
|
+
SET config = (
|
|
27
|
+
COALESCE(config, '{}'::jsonb)
|
|
28
|
+
- 'auto_approve_actions'
|
|
29
|
+
- 'require_approval_actions'
|
|
30
|
+
)
|
|
31
|
+
|| jsonb_build_object(
|
|
32
|
+
'action_modes',
|
|
33
|
+
COALESCE(
|
|
34
|
+
(
|
|
35
|
+
-- 'approval' wins over 'auto' when an op appears in both arrays
|
|
36
|
+
-- (MIN('approval', 'auto') = 'approval' lexicographically).
|
|
37
|
+
SELECT jsonb_object_agg(op_key, mode)
|
|
38
|
+
FROM (
|
|
39
|
+
SELECT op_key, MIN(mode) AS mode
|
|
40
|
+
FROM (
|
|
41
|
+
SELECT op_key, 'approval'::text AS mode
|
|
42
|
+
FROM jsonb_array_elements_text(
|
|
43
|
+
CASE
|
|
44
|
+
WHEN jsonb_typeof(config->'require_approval_actions') = 'array'
|
|
45
|
+
THEN config->'require_approval_actions'
|
|
46
|
+
ELSE '[]'::jsonb
|
|
47
|
+
END
|
|
48
|
+
) AS op_key
|
|
49
|
+
UNION ALL
|
|
50
|
+
SELECT op_key, 'auto'::text AS mode
|
|
51
|
+
FROM jsonb_array_elements_text(
|
|
52
|
+
CASE
|
|
53
|
+
WHEN jsonb_typeof(config->'auto_approve_actions') = 'array'
|
|
54
|
+
THEN config->'auto_approve_actions'
|
|
55
|
+
ELSE '[]'::jsonb
|
|
56
|
+
END
|
|
57
|
+
) AS op_key
|
|
58
|
+
) all_modes
|
|
59
|
+
GROUP BY op_key
|
|
60
|
+
) collapsed
|
|
61
|
+
),
|
|
62
|
+
'{}'::jsonb
|
|
63
|
+
)
|
|
64
|
+
)
|
|
65
|
+
WHERE config IS NOT NULL
|
|
66
|
+
AND (
|
|
67
|
+
config ? 'auto_approve_actions'
|
|
68
|
+
OR config ? 'require_approval_actions'
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
-- migrate:down
|
|
72
|
+
|
|
73
|
+
-- Reverse the collapse: split action_modes back into the two arrays.
|
|
74
|
+
-- 'auto' → auto_approve_actions
|
|
75
|
+
-- 'approval' → require_approval_actions
|
|
76
|
+
-- 'disabled' has no pre-refactor equivalent and is silently dropped on
|
|
77
|
+
-- downgrade — the agent will see the op again as if no override existed.
|
|
78
|
+
UPDATE public.connections
|
|
79
|
+
SET config = (
|
|
80
|
+
COALESCE(config, '{}'::jsonb) - 'action_modes'
|
|
81
|
+
)
|
|
82
|
+
|| jsonb_build_object(
|
|
83
|
+
'auto_approve_actions',
|
|
84
|
+
COALESCE(
|
|
85
|
+
(
|
|
86
|
+
SELECT jsonb_agg(key)
|
|
87
|
+
FROM jsonb_each_text(config->'action_modes')
|
|
88
|
+
WHERE value = 'auto'
|
|
89
|
+
),
|
|
90
|
+
'[]'::jsonb
|
|
91
|
+
),
|
|
92
|
+
'require_approval_actions',
|
|
93
|
+
COALESCE(
|
|
94
|
+
(
|
|
95
|
+
SELECT jsonb_agg(key)
|
|
96
|
+
FROM jsonb_each_text(config->'action_modes')
|
|
97
|
+
WHERE value = 'approval'
|
|
98
|
+
),
|
|
99
|
+
'[]'::jsonb
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
WHERE config IS NOT NULL
|
|
103
|
+
AND jsonb_typeof(config->'action_modes') = 'object';
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
-- Relax the device-binding XOR for browser_session profiles to allow
|
|
3
|
+
-- mirror mode, where neither user_data_dir nor cdp_url is set on the
|
|
4
|
+
-- row (the source profile dir lives in auth_data.source_profile_dir).
|
|
5
|
+
-- Keep the mutual exclusion of the two columns so they can't be set
|
|
6
|
+
-- together; application validation enforces "exactly one of mirror /
|
|
7
|
+
-- cdp / legacy" per row.
|
|
8
|
+
|
|
9
|
+
ALTER TABLE auth_profiles
|
|
10
|
+
DROP CONSTRAINT IF EXISTS auth_profiles_device_browser_path_xor;
|
|
11
|
+
|
|
12
|
+
ALTER TABLE auth_profiles
|
|
13
|
+
ADD CONSTRAINT auth_profiles_device_browser_path_mutex
|
|
14
|
+
CHECK (
|
|
15
|
+
device_worker_id IS NULL
|
|
16
|
+
OR profile_kind <> 'browser_session'
|
|
17
|
+
OR user_data_dir IS NULL
|
|
18
|
+
OR cdp_url IS NULL
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
-- migrate:down
|
|
22
|
+
ALTER TABLE auth_profiles
|
|
23
|
+
DROP CONSTRAINT IF EXISTS auth_profiles_device_browser_path_mutex;
|
|
24
|
+
|
|
25
|
+
ALTER TABLE auth_profiles
|
|
26
|
+
ADD CONSTRAINT auth_profiles_device_browser_path_xor
|
|
27
|
+
CHECK (
|
|
28
|
+
device_worker_id IS NULL
|
|
29
|
+
OR profile_kind <> 'browser_session'
|
|
30
|
+
OR ((user_data_dir IS NOT NULL) AND (cdp_url IS NULL))
|
|
31
|
+
OR ((user_data_dir IS NULL) AND (cdp_url IS NOT NULL))
|
|
32
|
+
);
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
-- Phase A of moving `agents` from a globally-unique `id` PK to a per-org
|
|
3
|
+
-- composite PK `(organization_id, id)`. The application has always treated
|
|
4
|
+
-- agents as org-scoped (every read/delete/list filters by organization_id),
|
|
5
|
+
-- so the global PK is a latent footgun: two orgs cannot share an agent ID,
|
|
6
|
+
-- and a stale agent in one org silently blocks another org from using the
|
|
7
|
+
-- same name.
|
|
8
|
+
--
|
|
9
|
+
-- This phase is INTENTIONALLY NON-BREAKING. It only:
|
|
10
|
+
-- 1. Adds an `organization_id` column to each FK-holding child table
|
|
11
|
+
-- (NULLABLE — backfilled here, set NOT NULL in a later phase once the
|
|
12
|
+
-- app-code refactor lands so every INSERT writes the value).
|
|
13
|
+
-- 2. Backfills `organization_id` from agents (no orphan rows in prod).
|
|
14
|
+
-- 3. Adds a parallel UNIQUE constraint on `agents (organization_id, id)`
|
|
15
|
+
-- so the schema is ready for the eventual PK swap.
|
|
16
|
+
-- 4. Adds composite indexes on each child table so the upcoming
|
|
17
|
+
-- org-scoped query patterns are fast from day one.
|
|
18
|
+
--
|
|
19
|
+
-- The single-column PK on agents and the single-column FKs on child tables
|
|
20
|
+
-- stay in place. App code keeps working unmodified. The PK swap and FK
|
|
21
|
+
-- composite migration ship in a separate PR after the storage interfaces
|
|
22
|
+
-- are plumbed with `organization_id`.
|
|
23
|
+
|
|
24
|
+
-- ── 1. Add organization_id columns (nullable for now).
|
|
25
|
+
ALTER TABLE agent_grants ADD COLUMN organization_id text;
|
|
26
|
+
ALTER TABLE agent_connections ADD COLUMN organization_id text;
|
|
27
|
+
ALTER TABLE agent_users ADD COLUMN organization_id text;
|
|
28
|
+
ALTER TABLE agent_channel_bindings ADD COLUMN organization_id text;
|
|
29
|
+
ALTER TABLE grants ADD COLUMN organization_id text;
|
|
30
|
+
|
|
31
|
+
-- ── 2. Backfill from agents.
|
|
32
|
+
UPDATE agent_grants SET organization_id = a.organization_id FROM agents a WHERE agent_grants.agent_id = a.id;
|
|
33
|
+
UPDATE agent_connections SET organization_id = a.organization_id FROM agents a WHERE agent_connections.agent_id = a.id;
|
|
34
|
+
UPDATE agent_users SET organization_id = a.organization_id FROM agents a WHERE agent_users.agent_id = a.id;
|
|
35
|
+
UPDATE agent_channel_bindings SET organization_id = a.organization_id FROM agents a WHERE agent_channel_bindings.agent_id = a.id;
|
|
36
|
+
UPDATE grants SET organization_id = a.organization_id FROM agents a WHERE grants.agent_id = a.id;
|
|
37
|
+
|
|
38
|
+
-- ── 3. Parallel UNIQUE on agents (organization_id, id). The single-column
|
|
39
|
+
-- PK on (id) stays — this is purely additive and signals to readers
|
|
40
|
+
-- that org-scoped uniqueness is the eventual model. The PK swap in a
|
|
41
|
+
-- later migration will drop this UNIQUE and reuse the index for the
|
|
42
|
+
-- new composite PK.
|
|
43
|
+
ALTER TABLE agents
|
|
44
|
+
ADD CONSTRAINT agents_organization_id_id_key UNIQUE (organization_id, id);
|
|
45
|
+
|
|
46
|
+
-- ── 4. Composite indexes on child tables for upcoming org-scoped queries.
|
|
47
|
+
CREATE INDEX agent_grants_org_agent_idx ON agent_grants (organization_id, agent_id);
|
|
48
|
+
CREATE INDEX agent_connections_org_agent_idx ON agent_connections (organization_id, agent_id);
|
|
49
|
+
CREATE INDEX agent_users_org_agent_idx ON agent_users (organization_id, agent_id);
|
|
50
|
+
CREATE INDEX agent_channel_bindings_org_agent_idx ON agent_channel_bindings (organization_id, agent_id);
|
|
51
|
+
CREATE INDEX grants_org_agent_idx ON grants (organization_id, agent_id);
|
|
52
|
+
|
|
53
|
+
-- migrate:down
|
|
54
|
+
DROP INDEX IF EXISTS grants_org_agent_idx;
|
|
55
|
+
DROP INDEX IF EXISTS agent_channel_bindings_org_agent_idx;
|
|
56
|
+
DROP INDEX IF EXISTS agent_users_org_agent_idx;
|
|
57
|
+
DROP INDEX IF EXISTS agent_connections_org_agent_idx;
|
|
58
|
+
DROP INDEX IF EXISTS agent_grants_org_agent_idx;
|
|
59
|
+
|
|
60
|
+
ALTER TABLE agents DROP CONSTRAINT IF EXISTS agents_organization_id_id_key;
|
|
61
|
+
|
|
62
|
+
ALTER TABLE grants DROP COLUMN IF EXISTS organization_id;
|
|
63
|
+
ALTER TABLE agent_channel_bindings DROP COLUMN IF EXISTS organization_id;
|
|
64
|
+
ALTER TABLE agent_users DROP COLUMN IF EXISTS organization_id;
|
|
65
|
+
ALTER TABLE agent_connections DROP COLUMN IF EXISTS organization_id;
|
|
66
|
+
ALTER TABLE agent_grants DROP COLUMN IF EXISTS organization_id;
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
|
|
3
|
+
-- =============================================================================
|
|
4
|
+
-- Geo enrichment: reverse-geocode lat/lng → country / admin1 / place at the
|
|
5
|
+
-- gateway, once, for every event with coordinates. Used by `apple.photos`
|
|
6
|
+
-- today; gmaps reviews, github commit metadata, and any future geo-bearing
|
|
7
|
+
-- connector benefit automatically.
|
|
8
|
+
--
|
|
9
|
+
-- Three system-level reference tables (no organization_id — these are
|
|
10
|
+
-- read-only geographic facts shared across all tenants) seeded from
|
|
11
|
+
-- GeoNames (https://www.geonames.org/, CC-BY 4.0):
|
|
12
|
+
--
|
|
13
|
+
-- geo_countries — country_code → name/continent/currency/etc. (~250 rows)
|
|
14
|
+
-- geo_admin1 — state/province codes per country (~4k rows)
|
|
15
|
+
-- geo_places — populated places (cities/towns/villages/hamlets);
|
|
16
|
+
-- seeded from GeoNames cities1000.txt (~150k rows for v1).
|
|
17
|
+
-- Can be upgraded to the full PPL-class subset of
|
|
18
|
+
-- allCountries (~5M rows) without schema changes —
|
|
19
|
+
-- nearest-neighbor query is the same shape.
|
|
20
|
+
--
|
|
21
|
+
-- The `geo_lookup(lat, lng)` function returns the enriched row in one
|
|
22
|
+
-- call. Nearest-neighbor uses PostGIS `geography(POINT, 4326)` + GiST so
|
|
23
|
+
-- distance is true geodesic (not L2-on-degrees), and the index keeps it
|
|
24
|
+
-- sub-millisecond at any table size we'd ever load.
|
|
25
|
+
--
|
|
26
|
+
-- Run `scripts/seed-geo-data.sh` after this migration applies to populate
|
|
27
|
+
-- the tables. The TS enrichment hook gracefully no-ops if the tables are
|
|
28
|
+
-- empty or the function is missing, so partially-deployed installs keep
|
|
29
|
+
-- working — events just don't get the enriched fields until seeding runs.
|
|
30
|
+
--
|
|
31
|
+
-- ENVIRONMENTS WITHOUT POSTGIS: the entire migration is wrapped in a DO
|
|
32
|
+
-- block that probes for the extension. If PostGIS isn't installable
|
|
33
|
+
-- (PGlite in tests, restricted hosts), every statement below is skipped
|
|
34
|
+
-- with a NOTICE. The runtime enrichment hook also fails open, so
|
|
35
|
+
-- partially-supported environments keep functioning — they just don't
|
|
36
|
+
-- get geo enrichment.
|
|
37
|
+
-- =============================================================================
|
|
38
|
+
|
|
39
|
+
DO $migration$
|
|
40
|
+
BEGIN
|
|
41
|
+
-- Try to install PostGIS. If it's not available on this host (PGlite
|
|
42
|
+
-- without the postgis extension registered, managed Postgres without
|
|
43
|
+
-- the extension, etc.), bail out cleanly.
|
|
44
|
+
BEGIN
|
|
45
|
+
CREATE EXTENSION IF NOT EXISTS postgis;
|
|
46
|
+
EXCEPTION
|
|
47
|
+
WHEN OTHERS THEN
|
|
48
|
+
RAISE NOTICE
|
|
49
|
+
'geo-enrichment: PostGIS unavailable (%), skipping geo schema. Runtime enrichment will no-op.',
|
|
50
|
+
SQLERRM;
|
|
51
|
+
RETURN;
|
|
52
|
+
END;
|
|
53
|
+
|
|
54
|
+
-- spatial_ref_sys row for SRID 4326 (WGS-84). Real PostGIS installs
|
|
55
|
+
-- bundle ~8000 standard projections; the pglite-postgis WASM build
|
|
56
|
+
-- ships an empty table to keep the bundle small. Inserting the one
|
|
57
|
+
-- row we use makes nearest-neighbour queries work everywhere; the
|
|
58
|
+
-- ON CONFLICT skips on prod where the row already exists.
|
|
59
|
+
INSERT INTO spatial_ref_sys (srid, auth_name, auth_srid, srtext, proj4text)
|
|
60
|
+
VALUES (
|
|
61
|
+
4326,
|
|
62
|
+
'EPSG',
|
|
63
|
+
4326,
|
|
64
|
+
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
|
|
65
|
+
'+proj=longlat +datum=WGS84 +no_defs'
|
|
66
|
+
)
|
|
67
|
+
ON CONFLICT (srid) DO NOTHING;
|
|
68
|
+
|
|
69
|
+
-- Everything below this point assumes PostGIS is loaded. EXECUTE-wrapping
|
|
70
|
+
-- the DDL keeps the SQL parser from choking on `geography(POINT, 4326)`
|
|
71
|
+
-- when this whole DO block is parsed before the extension creates the type.
|
|
72
|
+
|
|
73
|
+
-- geo_countries — ISO-2 country code → full record. Source: GeoNames
|
|
74
|
+
-- countryInfo.txt.
|
|
75
|
+
EXECUTE $ddl$
|
|
76
|
+
CREATE TABLE IF NOT EXISTS geo_countries (
|
|
77
|
+
code text PRIMARY KEY,
|
|
78
|
+
code3 text,
|
|
79
|
+
numeric_code integer,
|
|
80
|
+
fips text,
|
|
81
|
+
name text NOT NULL,
|
|
82
|
+
capital text,
|
|
83
|
+
area_sq_km numeric,
|
|
84
|
+
population bigint,
|
|
85
|
+
continent text,
|
|
86
|
+
tld text,
|
|
87
|
+
currency_code text,
|
|
88
|
+
currency_name text,
|
|
89
|
+
phone text,
|
|
90
|
+
postal_code_fmt text,
|
|
91
|
+
postal_code_re text,
|
|
92
|
+
languages text,
|
|
93
|
+
geonameid bigint,
|
|
94
|
+
neighbours text
|
|
95
|
+
)
|
|
96
|
+
$ddl$;
|
|
97
|
+
|
|
98
|
+
-- geo_admin1 — first-order administrative subdivisions.
|
|
99
|
+
-- Code shape: '<ISO2>.<ADMIN1>' (e.g. 'IT.07' = Lazio).
|
|
100
|
+
EXECUTE $ddl$
|
|
101
|
+
CREATE TABLE IF NOT EXISTS geo_admin1 (
|
|
102
|
+
code text PRIMARY KEY,
|
|
103
|
+
country_code text NOT NULL,
|
|
104
|
+
name text NOT NULL,
|
|
105
|
+
ascii_name text NOT NULL,
|
|
106
|
+
geonameid bigint
|
|
107
|
+
)
|
|
108
|
+
$ddl$;
|
|
109
|
+
|
|
110
|
+
EXECUTE 'CREATE INDEX IF NOT EXISTS geo_admin1_country_idx ON geo_admin1 (country_code)';
|
|
111
|
+
|
|
112
|
+
-- geo_places — populated places. `location` is a generated geography
|
|
113
|
+
-- point that the GiST index uses for nearest-neighbour lookup. Stays
|
|
114
|
+
-- sub-ms even at 5M+ rows.
|
|
115
|
+
EXECUTE $ddl$
|
|
116
|
+
CREATE TABLE IF NOT EXISTS geo_places (
|
|
117
|
+
geonameid bigint PRIMARY KEY,
|
|
118
|
+
name text NOT NULL,
|
|
119
|
+
ascii_name text NOT NULL,
|
|
120
|
+
alt_names text,
|
|
121
|
+
latitude double precision NOT NULL,
|
|
122
|
+
longitude double precision NOT NULL,
|
|
123
|
+
feature_class text,
|
|
124
|
+
feature_code text,
|
|
125
|
+
country_code text NOT NULL,
|
|
126
|
+
admin1_code text,
|
|
127
|
+
admin2_code text,
|
|
128
|
+
population bigint DEFAULT 0,
|
|
129
|
+
elevation_m integer,
|
|
130
|
+
timezone text,
|
|
131
|
+
location geography(POINT, 4326)
|
|
132
|
+
GENERATED ALWAYS AS (
|
|
133
|
+
ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography
|
|
134
|
+
) STORED
|
|
135
|
+
)
|
|
136
|
+
$ddl$;
|
|
137
|
+
|
|
138
|
+
EXECUTE 'CREATE INDEX IF NOT EXISTS geo_places_location_idx ON geo_places USING GIST (location)';
|
|
139
|
+
EXECUTE 'CREATE INDEX IF NOT EXISTS geo_places_country_idx ON geo_places (country_code)';
|
|
140
|
+
|
|
141
|
+
-- geo_lookup(lat, lng) — single-call enrichment.
|
|
142
|
+
-- Returns the nearest populated place plus the country/admin1 join.
|
|
143
|
+
-- distance_km is included so callers can apply their own threshold
|
|
144
|
+
-- (e.g., reject results > 500 km away — ocean/desert coordinates that
|
|
145
|
+
-- would otherwise snap misleadingly to the closest coastal city).
|
|
146
|
+
EXECUTE $fn$
|
|
147
|
+
CREATE OR REPLACE FUNCTION geo_lookup(p_lat double precision, p_lng double precision)
|
|
148
|
+
RETURNS TABLE (
|
|
149
|
+
place_name text,
|
|
150
|
+
place_id bigint,
|
|
151
|
+
country_code text,
|
|
152
|
+
country_name text,
|
|
153
|
+
admin1_code text,
|
|
154
|
+
admin1_name text,
|
|
155
|
+
timezone text,
|
|
156
|
+
population bigint,
|
|
157
|
+
distance_km double precision
|
|
158
|
+
)
|
|
159
|
+
LANGUAGE sql
|
|
160
|
+
STABLE
|
|
161
|
+
PARALLEL SAFE
|
|
162
|
+
AS $body$
|
|
163
|
+
WITH nearest AS (
|
|
164
|
+
SELECT
|
|
165
|
+
p.geonameid,
|
|
166
|
+
p.name,
|
|
167
|
+
p.country_code,
|
|
168
|
+
p.admin1_code,
|
|
169
|
+
p.timezone,
|
|
170
|
+
p.population,
|
|
171
|
+
ST_Distance(
|
|
172
|
+
p.location,
|
|
173
|
+
ST_SetSRID(ST_MakePoint(p_lng, p_lat), 4326)::geography
|
|
174
|
+
) / 1000.0 AS distance_km
|
|
175
|
+
FROM geo_places p
|
|
176
|
+
ORDER BY p.location <-> ST_SetSRID(ST_MakePoint(p_lng, p_lat), 4326)::geography
|
|
177
|
+
LIMIT 1
|
|
178
|
+
)
|
|
179
|
+
SELECT
|
|
180
|
+
n.name AS place_name,
|
|
181
|
+
n.geonameid AS place_id,
|
|
182
|
+
n.country_code AS country_code,
|
|
183
|
+
c.name AS country_name,
|
|
184
|
+
CASE
|
|
185
|
+
WHEN n.admin1_code IS NULL OR n.admin1_code = '' THEN NULL
|
|
186
|
+
ELSE n.country_code || '.' || n.admin1_code
|
|
187
|
+
END AS admin1_code,
|
|
188
|
+
a.name AS admin1_name,
|
|
189
|
+
n.timezone AS timezone,
|
|
190
|
+
n.population AS population,
|
|
191
|
+
n.distance_km AS distance_km
|
|
192
|
+
FROM nearest n
|
|
193
|
+
LEFT JOIN geo_countries c ON c.code = n.country_code
|
|
194
|
+
LEFT JOIN geo_admin1 a ON a.code = n.country_code || '.' || n.admin1_code
|
|
195
|
+
$body$
|
|
196
|
+
$fn$;
|
|
197
|
+
END
|
|
198
|
+
$migration$;
|
|
199
|
+
|
|
200
|
+
-- migrate:down
|
|
201
|
+
|
|
202
|
+
DROP FUNCTION IF EXISTS geo_lookup(double precision, double precision);
|
|
203
|
+
DROP TABLE IF EXISTS geo_places;
|
|
204
|
+
DROP TABLE IF EXISTS geo_admin1;
|
|
205
|
+
DROP TABLE IF EXISTS geo_countries;
|
|
206
|
+
-- Intentionally do NOT DROP EXTENSION postgis. The extension may be
|
|
207
|
+
-- shared by other tables / future migrations on the same Postgres
|
|
208
|
+
-- instance; rolling back this migration shouldn't take that down.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
-- Drop the parallel `UNIQUE (organization_id, id)` added in 20260515120000.
|
|
3
|
+
-- It was meant as schema-prep for the eventual PK swap to (organization_id,
|
|
4
|
+
-- id), but it actively broke `ON CONFLICT (id) DO NOTHING/UPDATE` callers.
|
|
5
|
+
--
|
|
6
|
+
-- Why: Postgres' `ON CONFLICT (X)` only suppresses violations of the unique
|
|
7
|
+
-- constraint matching exactly column set X. Adding a second unique constraint
|
|
8
|
+
-- that overlaps with the PK means inserts can fail on the new constraint
|
|
9
|
+
-- before reaching the PK conflict — and ON CONFLICT (id) doesn't catch it.
|
|
10
|
+
-- Surfaced in `__tests__/integration/.../race-mcp` where parallel inserts of
|
|
11
|
+
-- `(org-a, race-mcp-0)` started throwing `agents_organization_id_id_key`
|
|
12
|
+
-- duplicates instead of being silently de-duped by the existing
|
|
13
|
+
-- `ON CONFLICT (id) DO NOTHING` clause.
|
|
14
|
+
--
|
|
15
|
+
-- The PK on `(id)` already enforces global uniqueness, which subsumes
|
|
16
|
+
-- `(organization_id, id)` uniqueness — the new constraint was logically
|
|
17
|
+
-- redundant. Phase C of the per-org PK migration will swap the PK directly
|
|
18
|
+
-- without needing a parallel constraint as a stepping stone.
|
|
19
|
+
|
|
20
|
+
ALTER TABLE agents DROP CONSTRAINT IF EXISTS agents_organization_id_id_key;
|
|
21
|
+
|
|
22
|
+
-- migrate:down
|
|
23
|
+
ALTER TABLE agents
|
|
24
|
+
ADD CONSTRAINT agents_organization_id_id_key UNIQUE (organization_id, id);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
-- migrate:up
|
|
2
|
+
-- Admin-managed default app profile per (org, connector_key).
|
|
3
|
+
-- Today getPrimaryAuthProfileForKind picks the most-recently-updated active
|
|
4
|
+
-- oauth_app profile for the connector — admins have no way to designate
|
|
5
|
+
-- which profile members should fall through to. The flag lets the admin
|
|
6
|
+
-- pin a chosen profile; the resolver prefers flagged rows first.
|
|
7
|
+
--
|
|
8
|
+
-- Constrained to oauth_app for now since that's the only kind where
|
|
9
|
+
-- "default for connector" is meaningful (env / interactive / browser_session
|
|
10
|
+
-- are picked by other rules — device binding, capture mode, etc.).
|
|
11
|
+
|
|
12
|
+
ALTER TABLE auth_profiles
|
|
13
|
+
ADD COLUMN is_default_for_connector boolean NOT NULL DEFAULT false;
|
|
14
|
+
|
|
15
|
+
CREATE UNIQUE INDEX auth_profiles_default_for_connector_unique
|
|
16
|
+
ON auth_profiles (organization_id, connector_key)
|
|
17
|
+
WHERE is_default_for_connector AND profile_kind = 'oauth_app';
|
|
18
|
+
|
|
19
|
+
-- migrate:down
|
|
20
|
+
DROP INDEX IF EXISTS auth_profiles_default_for_connector_unique;
|
|
21
|
+
|
|
22
|
+
ALTER TABLE auth_profiles
|
|
23
|
+
DROP COLUMN IF EXISTS is_default_for_connector;
|