opensentinel 2.1.1 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +354 -283
- package/dist/archiver-AVNBYCKQ.js +15340 -0
- package/dist/archiver-AVNBYCKQ.js.map +1 -0
- package/dist/audit-logger-OBPR7CRO.js +22 -0
- package/dist/auth-UOX5K2BE.js +18 -0
- package/dist/autonomy-ZXDBDQUJ.js +86 -0
- package/dist/autonomy-ZXDBDQUJ.js.map +1 -0
- package/dist/aws-s3-Q4LLZZPD.js +146 -0
- package/dist/aws-s3-Q4LLZZPD.js.map +1 -0
- package/dist/backup-restore-PZ7CYYB7.js +16 -0
- package/dist/blocks-R3PODY47.js +23 -0
- package/dist/bot-QRARP4UN.js +36 -0
- package/dist/brain-7XLLM3KC.js +56 -0
- package/dist/camera-monitor-M5CYKUU4.js +335 -0
- package/dist/camera-monitor-M5CYKUU4.js.map +1 -0
- package/dist/{charts-MMXM6BWW.js → charts-V7ARZNKF.js} +2 -2
- package/dist/chunk-22VGGA7S.js +330 -0
- package/dist/chunk-22VGGA7S.js.map +1 -0
- package/dist/chunk-35WYTA3C.js +382 -0
- package/dist/chunk-35WYTA3C.js.map +1 -0
- package/dist/chunk-3E2PSU2C.js +146 -0
- package/dist/chunk-3E2PSU2C.js.map +1 -0
- package/dist/{chunk-L3F43VPB.js → chunk-4GLYY4NN.js} +2 -2
- package/dist/{chunk-L3F43VPB.js.map → chunk-4GLYY4NN.js.map} +1 -1
- package/dist/{chunk-L3PDU3XN.js → chunk-4UOE5TUZ.js} +4 -4
- package/dist/{chunk-6SNHU3CY.js → chunk-66OJ3WB4.js} +2 -2
- package/dist/chunk-6KONMXQ6.js +297 -0
- package/dist/chunk-6KONMXQ6.js.map +1 -0
- package/dist/chunk-6PMVAAA7.js +196 -0
- package/dist/chunk-6PMVAAA7.js.map +1 -0
- package/dist/chunk-766ASQWE.js +32620 -0
- package/dist/chunk-766ASQWE.js.map +1 -0
- package/dist/chunk-7WQO5J2M.js +29 -0
- package/dist/chunk-7WQO5J2M.js.map +1 -0
- package/dist/chunk-APHSRMBS.js +148 -0
- package/dist/chunk-APHSRMBS.js.map +1 -0
- package/dist/{chunk-4LVWXUNC.js → chunk-AYUKPTSM.js} +57 -39
- package/dist/chunk-AYUKPTSM.js.map +1 -0
- package/dist/chunk-BIPYADGB.js +84 -0
- package/dist/chunk-BIPYADGB.js.map +1 -0
- package/dist/chunk-BRBWNV65.js +457 -0
- package/dist/chunk-BRBWNV65.js.map +1 -0
- package/dist/chunk-BXZ6EA52.js +382 -0
- package/dist/chunk-BXZ6EA52.js.map +1 -0
- package/dist/chunk-EVE7MIIY.js +290 -0
- package/dist/chunk-EVE7MIIY.js.map +1 -0
- package/dist/chunk-F3TTNID2.js +138 -0
- package/dist/chunk-F3TTNID2.js.map +1 -0
- package/dist/chunk-H5RQOFO2.js +190 -0
- package/dist/chunk-H5RQOFO2.js.map +1 -0
- package/dist/chunk-HN3F4WSW.js +145 -0
- package/dist/chunk-HN3F4WSW.js.map +1 -0
- package/dist/{chunk-6DRDKB45.js → chunk-I6BDYQIG.js} +20 -9
- package/dist/chunk-I6BDYQIG.js.map +1 -0
- package/dist/chunk-IZJMVV7O.js +347 -0
- package/dist/chunk-IZJMVV7O.js.map +1 -0
- package/dist/chunk-KM22GV7G.js +211 -0
- package/dist/chunk-KM22GV7G.js.map +1 -0
- package/dist/chunk-MGFBLVR7.js +103 -0
- package/dist/chunk-MGFBLVR7.js.map +1 -0
- package/dist/chunk-MQJ2ECQT.js +228 -0
- package/dist/chunk-MQJ2ECQT.js.map +1 -0
- package/dist/{chunk-F6QUZQGI.js → chunk-MXAPLSJ5.js} +2 -2
- package/dist/{chunk-GK3E2I7A.js → chunk-NHMBTUMW.js} +2 -2
- package/dist/chunk-NPRTSZIF.js +131 -0
- package/dist/chunk-NPRTSZIF.js.map +1 -0
- package/dist/chunk-O7IH7JTI.js +1898 -0
- package/dist/chunk-O7IH7JTI.js.map +1 -0
- package/dist/chunk-OCVQGBJK.js +293 -0
- package/dist/chunk-OCVQGBJK.js.map +1 -0
- package/dist/chunk-P6QINGFL.js +332 -0
- package/dist/chunk-P6QINGFL.js.map +1 -0
- package/dist/chunk-PHDZKPNE.js +91 -0
- package/dist/chunk-PHDZKPNE.js.map +1 -0
- package/dist/chunk-PLDDJCW6.js +49 -0
- package/dist/chunk-PTGTGXV2.js +164 -0
- package/dist/chunk-PTGTGXV2.js.map +1 -0
- package/dist/chunk-REMIY4U2.js +171 -0
- package/dist/chunk-REMIY4U2.js.map +1 -0
- package/dist/chunk-RZ4YESBG.js +141 -0
- package/dist/chunk-RZ4YESBG.js.map +1 -0
- package/dist/chunk-SAX5MHK4.js +111 -0
- package/dist/chunk-SAX5MHK4.js.map +1 -0
- package/dist/{chunk-GVJVEWHI.js → chunk-SJSUSJ47.js} +2 -2
- package/dist/chunk-SPPMCAKG.js +777 -0
- package/dist/chunk-SPPMCAKG.js.map +1 -0
- package/dist/chunk-SVAPX2XN.js +2441 -0
- package/dist/chunk-SVAPX2XN.js.map +1 -0
- package/dist/chunk-TVEWKIK3.js +452 -0
- package/dist/chunk-TVEWKIK3.js.map +1 -0
- package/dist/{chunk-HH2HBTQM.js → chunk-TYAGMJNV.js} +5 -5
- package/dist/{chunk-JXUP2X7V.js → chunk-VEHFVBLI.js} +2 -2
- package/dist/chunk-VNX5GMTN.js +128 -0
- package/dist/chunk-VNX5GMTN.js.map +1 -0
- package/dist/chunk-VRD5CYRL.js +1568 -0
- package/dist/chunk-VRD5CYRL.js.map +1 -0
- package/dist/chunk-WLUHNG6X.js +122 -0
- package/dist/chunk-WLUHNG6X.js.map +1 -0
- package/dist/chunk-WRAKK6K6.js +265 -0
- package/dist/chunk-WRAKK6K6.js.map +1 -0
- package/dist/chunk-XKYRH4FM.js +681 -0
- package/dist/chunk-XKYRH4FM.js.map +1 -0
- package/dist/{chunk-GUBEEYDW.js → chunk-XMCVRVTF.js} +2 -2
- package/dist/{chunk-GUBEEYDW.js.map → chunk-XMCVRVTF.js.map} +1 -1
- package/dist/chunk-ZLZKF2PM.js +310 -0
- package/dist/chunk-ZLZKF2PM.js.map +1 -0
- package/dist/cli.js +5 -1
- package/dist/cli.js.map +1 -1
- package/dist/client-ZQSFPMOB.js +21 -0
- package/dist/clipboard-manager-TEO2GEDN.js +24 -0
- package/dist/commands/setup.js +3 -3
- package/dist/commands/setup.js.map +1 -1
- package/dist/commands/start.js +3 -3
- package/dist/commands/status.js +2 -2
- package/dist/commands/stop.js +2 -2
- package/dist/commands/utils.js +2 -2
- package/dist/cron-explain-HHQKPD3M.js +16 -0
- package/dist/crypto-4AP47IKC.js +14 -0
- package/dist/crypto-4AP47IKC.js.map +1 -0
- package/dist/databases-37X4CI2Y.js +21 -0
- package/dist/databases-37X4CI2Y.js.map +1 -0
- package/dist/discord-B3HUPGQ6.js +70 -0
- package/dist/discord-B3HUPGQ6.js.map +1 -0
- package/dist/dist-UISMLMFN.js +21847 -0
- package/dist/dist-UISMLMFN.js.map +1 -0
- package/dist/email-K7LO2IPB.js +268 -0
- package/dist/email-K7LO2IPB.js.map +1 -0
- package/dist/enhanced-retrieval-DNLLEM4Z.js +753 -0
- package/dist/enhanced-retrieval-DNLLEM4Z.js.map +1 -0
- package/dist/enrichment-pipeline-MNHNW65K.js +13 -0
- package/dist/enrichment-pipeline-MNHNW65K.js.map +1 -0
- package/dist/entity-resolution-Y3IUWEAT.js +24 -0
- package/dist/entity-resolution-Y3IUWEAT.js.map +1 -0
- package/dist/env-IWXUVTCB.js +12 -0
- package/dist/env-IWXUVTCB.js.map +1 -0
- package/dist/google-workspace-DKWUVNGC.js +169 -0
- package/dist/google-workspace-DKWUVNGC.js.map +1 -0
- package/dist/hash-tool-ULQYD7B5.js +22 -0
- package/dist/hash-tool-ULQYD7B5.js.map +1 -0
- package/dist/heartbeat-monitor-GCISLXI3.js +22 -0
- package/dist/heartbeat-monitor-GCISLXI3.js.map +1 -0
- package/dist/image-generation-OSU7FP6F.js +486 -0
- package/dist/image-generation-OSU7FP6F.js.map +1 -0
- package/dist/imessage-NGA2XF2V.js +35 -0
- package/dist/imessage-NGA2XF2V.js.map +1 -0
- package/dist/inbox-summarizer-NRI4S7IF.js +47 -0
- package/dist/inbox-summarizer-NRI4S7IF.js.map +1 -0
- package/dist/incident-response-C5J7Q6DT.js +244 -0
- package/dist/incident-response-C5J7Q6DT.js.map +1 -0
- package/dist/inventory-manager-352OHXWD.js +24 -0
- package/dist/inventory-manager-352OHXWD.js.map +1 -0
- package/dist/jira-GSGDBMIG.js +199 -0
- package/dist/jira-GSGDBMIG.js.map +1 -0
- package/dist/json-tool-QE2SYHEG.js +26 -0
- package/dist/json-tool-QE2SYHEG.js.map +1 -0
- package/dist/key-rotation-DPHU4ZTB.js +18 -0
- package/dist/key-rotation-DPHU4ZTB.js.map +1 -0
- package/dist/lib.d.ts +603 -11
- package/dist/lib.js +161 -35
- package/dist/lib.js.map +1 -1
- package/dist/mailchimp-KKNF6QJ7.js +152 -0
- package/dist/mailchimp-KKNF6QJ7.js.map +1 -0
- package/dist/matrix-QVHG76I7.js +279 -0
- package/dist/matrix-QVHG76I7.js.map +1 -0
- package/dist/{mcp-LS7Q3Z5W.js → mcp-3JI6W7ZE.js} +3 -3
- package/dist/mcp-3JI6W7ZE.js.map +1 -0
- package/dist/microsoft365-UCBKJHNX.js +164 -0
- package/dist/microsoft365-UCBKJHNX.js.map +1 -0
- package/dist/ocr-AC7NPX33.js +22 -0
- package/dist/ocr-AC7NPX33.js.map +1 -0
- package/dist/ollama-BOAMSPLJ.js +8 -0
- package/dist/ollama-BOAMSPLJ.js.map +1 -0
- package/dist/pages-MI523RB7.js +26 -0
- package/dist/pages-MI523RB7.js.map +1 -0
- package/dist/pair-JDFTERIK.js +24 -0
- package/dist/pair-JDFTERIK.js.map +1 -0
- package/dist/pairing-IFQYCPNS.js +10 -0
- package/dist/pairing-IFQYCPNS.js.map +1 -0
- package/dist/pdf-ALQVOEJR.js +17 -0
- package/dist/pdf-ALQVOEJR.js.map +1 -0
- package/dist/presentations-DSV5IHG5.js +1002 -0
- package/dist/presentations-DSV5IHG5.js.map +1 -0
- package/dist/prometheus-JNT2BD4L.js +10 -0
- package/dist/prometheus-JNT2BD4L.js.map +1 -0
- package/dist/providers-J4LYPHDR.js +19 -0
- package/dist/providers-J4LYPHDR.js.map +1 -0
- package/dist/qr-code-WIX4PB4U.js +16 -0
- package/dist/qr-code-WIX4PB4U.js.map +1 -0
- package/dist/quickbooks-XB4NII2S.js +190 -0
- package/dist/quickbooks-XB4NII2S.js.map +1 -0
- package/dist/regex-tool-W4ABRKGK.js +24 -0
- package/dist/regex-tool-W4ABRKGK.js.map +1 -0
- package/dist/scheduler-VK4WFERV.js +63 -0
- package/dist/scheduler-VK4WFERV.js.map +1 -0
- package/dist/search-BCLBO5E3.js +25 -0
- package/dist/search-BCLBO5E3.js.map +1 -0
- package/dist/sendgrid-RNXCAFKM.js +152 -0
- package/dist/sendgrid-RNXCAFKM.js.map +1 -0
- package/dist/shopify-NCXYJB4R.js +171 -0
- package/dist/shopify-NCXYJB4R.js.map +1 -0
- package/dist/signal-6CGDFYL2.js +35 -0
- package/dist/signal-6CGDFYL2.js.map +1 -0
- package/dist/slack-IZQWIKOH.js +75 -0
- package/dist/slack-IZQWIKOH.js.map +1 -0
- package/dist/sms-M3JIOTCW.js +23 -0
- package/dist/sms-M3JIOTCW.js.map +1 -0
- package/dist/{src-K7GASHRH.js → src-VYUE6LRA.js} +138 -32
- package/dist/src-VYUE6LRA.js.map +1 -0
- package/dist/stocks-XXWBPOCU.js +14 -0
- package/dist/stocks-XXWBPOCU.js.map +1 -0
- package/dist/text-transform-6SGUA5Z4.js +22 -0
- package/dist/text-transform-6SGUA5Z4.js.map +1 -0
- package/dist/tools-2RLEI2N6.js +38 -0
- package/dist/tools-2RLEI2N6.js.map +1 -0
- package/dist/tunnel-IWMXUML4.js +301 -0
- package/dist/tunnel-IWMXUML4.js.map +1 -0
- package/dist/twilio-53GEW5JT.js +139 -0
- package/dist/twilio-53GEW5JT.js.map +1 -0
- package/dist/unit-converter-ZYXMEZOE.js +14 -0
- package/dist/unit-converter-ZYXMEZOE.js.map +1 -0
- package/dist/whatsapp-LFX6YKCM.js +35 -0
- package/dist/whatsapp-LFX6YKCM.js.map +1 -0
- package/dist/word-document-7B6SJMAY.js +902 -0
- package/dist/word-document-7B6SJMAY.js.map +1 -0
- package/dist/xero-QYO66D45.js +162 -0
- package/dist/xero-QYO66D45.js.map +1 -0
- package/dist/zapier-webhook-TBZ5YF2A.js +106 -0
- package/dist/zapier-webhook-TBZ5YF2A.js.map +1 -0
- package/drizzle/0002_mushy_master_mold.sql +140 -0
- package/drizzle/meta/0002_snapshot.json +3637 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +100 -98
- package/dist/bot-KJ26BG56.js +0 -15
- package/dist/chunk-4LVWXUNC.js.map +0 -1
- package/dist/chunk-4TG2IG5K.js +0 -5249
- package/dist/chunk-4TG2IG5K.js.map +0 -1
- package/dist/chunk-6DRDKB45.js.map +0 -1
- package/dist/chunk-CI6Q63MM.js +0 -1613
- package/dist/chunk-CI6Q63MM.js.map +0 -1
- package/dist/chunk-KHNYJY2Z.js +0 -178
- package/dist/chunk-KHNYJY2Z.js.map +0 -1
- package/dist/chunk-NSBPE2FW.js +0 -17
- package/dist/discord-ZOJFTVTB.js +0 -49
- package/dist/imessage-JFRB6EJ7.js +0 -14
- package/dist/scheduler-EZ7CZMCS.js +0 -42
- package/dist/signal-T3MCSULM.js +0 -14
- package/dist/slack-N2M4FHAJ.js +0 -54
- package/dist/src-K7GASHRH.js.map +0 -1
- package/dist/tools-24GZHYRF.js +0 -16
- package/dist/whatsapp-VCRUPAO5.js +0 -14
- /package/dist/{bot-KJ26BG56.js.map → audit-logger-OBPR7CRO.js.map} +0 -0
- /package/dist/{chunk-NSBPE2FW.js.map → auth-UOX5K2BE.js.map} +0 -0
- /package/dist/{discord-ZOJFTVTB.js.map → backup-restore-PZ7CYYB7.js.map} +0 -0
- /package/dist/{imessage-JFRB6EJ7.js.map → blocks-R3PODY47.js.map} +0 -0
- /package/dist/{mcp-LS7Q3Z5W.js.map → bot-QRARP4UN.js.map} +0 -0
- /package/dist/{scheduler-EZ7CZMCS.js.map → brain-7XLLM3KC.js.map} +0 -0
- /package/dist/{charts-MMXM6BWW.js.map → charts-V7ARZNKF.js.map} +0 -0
- /package/dist/{chunk-L3PDU3XN.js.map → chunk-4UOE5TUZ.js.map} +0 -0
- /package/dist/{chunk-6SNHU3CY.js.map → chunk-66OJ3WB4.js.map} +0 -0
- /package/dist/{chunk-F6QUZQGI.js.map → chunk-MXAPLSJ5.js.map} +0 -0
- /package/dist/{chunk-GK3E2I7A.js.map → chunk-NHMBTUMW.js.map} +0 -0
- /package/dist/{signal-T3MCSULM.js.map → chunk-PLDDJCW6.js.map} +0 -0
- /package/dist/{chunk-GVJVEWHI.js.map → chunk-SJSUSJ47.js.map} +0 -0
- /package/dist/{chunk-HH2HBTQM.js.map → chunk-TYAGMJNV.js.map} +0 -0
- /package/dist/{chunk-JXUP2X7V.js.map → chunk-VEHFVBLI.js.map} +0 -0
- /package/dist/{slack-N2M4FHAJ.js.map → client-ZQSFPMOB.js.map} +0 -0
- /package/dist/{tools-24GZHYRF.js.map → clipboard-manager-TEO2GEDN.js.map} +0 -0
- /package/dist/{whatsapp-VCRUPAO5.js.map → cron-explain-HHQKPD3M.js.map} +0 -0
|
@@ -0,0 +1,2441 @@
|
|
|
1
|
+
import {
|
|
2
|
+
resolveEntity
|
|
3
|
+
} from "./chunk-WRAKK6K6.js";
|
|
4
|
+
import {
|
|
5
|
+
db,
|
|
6
|
+
graphEntities,
|
|
7
|
+
graphRelationships
|
|
8
|
+
} from "./chunk-XKYRH4FM.js";
|
|
9
|
+
import {
|
|
10
|
+
env
|
|
11
|
+
} from "./chunk-ZLZKF2PM.js";
|
|
12
|
+
|
|
13
|
+
// src/core/intelligence/enrichment-pipeline.ts
|
|
14
|
+
import { eq as eq2, sql } from "drizzle-orm";
|
|
15
|
+
|
|
16
|
+
// src/integrations/public-records/rate-limiter.ts
|
|
17
|
+
var RateLimiter = class {
|
|
18
|
+
constructor(name, maxPerWindow, windowMs) {
|
|
19
|
+
this.name = name;
|
|
20
|
+
this.maxPerWindow = maxPerWindow;
|
|
21
|
+
this.windowMs = windowMs;
|
|
22
|
+
this.bufferMs = Number(env.OSINT_RATE_LIMIT_BUFFER_MS) || 200;
|
|
23
|
+
}
|
|
24
|
+
/** Timestamps (ms) of requests within the current window */
|
|
25
|
+
timestamps = [];
|
|
26
|
+
/** Global buffer added after each wait to avoid edge-of-window bursts */
|
|
27
|
+
bufferMs;
|
|
28
|
+
/**
|
|
29
|
+
* Acquire permission to make a request.
|
|
30
|
+
* Resolves immediately if within limits, otherwise waits until a slot opens.
|
|
31
|
+
*/
|
|
32
|
+
async acquire() {
|
|
33
|
+
this.pruneExpired();
|
|
34
|
+
if (this.timestamps.length < this.maxPerWindow) {
|
|
35
|
+
this.timestamps.push(Date.now());
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const oldest = this.timestamps[0];
|
|
39
|
+
const waitMs = oldest + this.windowMs - Date.now() + this.bufferMs;
|
|
40
|
+
if (waitMs > 0) {
|
|
41
|
+
console.log(
|
|
42
|
+
`[OSINT:RateLimiter:${this.name}] Rate limit reached (${this.timestamps.length}/${this.maxPerWindow}), waiting ${waitMs}ms`
|
|
43
|
+
);
|
|
44
|
+
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
45
|
+
}
|
|
46
|
+
this.pruneExpired();
|
|
47
|
+
this.timestamps.push(Date.now());
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Number of requests remaining in the current window.
|
|
51
|
+
*/
|
|
52
|
+
get remaining() {
|
|
53
|
+
this.pruneExpired();
|
|
54
|
+
return Math.max(0, this.maxPerWindow - this.timestamps.length);
|
|
55
|
+
}
|
|
56
|
+
/** Remove timestamps that have fallen outside the window */
|
|
57
|
+
pruneExpired() {
|
|
58
|
+
const cutoff = Date.now() - this.windowMs;
|
|
59
|
+
while (this.timestamps.length > 0 && this.timestamps[0] < cutoff) {
|
|
60
|
+
this.timestamps.shift();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
function createRateLimiter(name, maxPerWindow, windowMs) {
|
|
65
|
+
return new RateLimiter(name, maxPerWindow, windowMs);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// src/integrations/public-records/fec-client.ts
|
|
69
|
+
var FECClientError = class extends Error {
|
|
70
|
+
constructor(message, statusCode) {
|
|
71
|
+
super(message);
|
|
72
|
+
this.statusCode = statusCode;
|
|
73
|
+
this.name = "FECClientError";
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
var LOG_PREFIX = "[OSINT:FEC]";
|
|
77
|
+
var FECClient = class {
|
|
78
|
+
baseUrl = "https://api.open.fec.gov/v1";
|
|
79
|
+
apiKey;
|
|
80
|
+
timeout;
|
|
81
|
+
maxPages;
|
|
82
|
+
rateLimiter;
|
|
83
|
+
constructor(config = {}) {
|
|
84
|
+
this.apiKey = config.apiKey ?? env.FEC_API_KEY ?? "";
|
|
85
|
+
this.timeout = config.timeout ?? 15e3;
|
|
86
|
+
this.maxPages = config.maxPages ?? 10;
|
|
87
|
+
this.rateLimiter = createRateLimiter("FEC", 1e3, 60 * 60 * 1e3);
|
|
88
|
+
}
|
|
89
|
+
// -----------------------------------------------------------------------
|
|
90
|
+
// Internal helpers
|
|
91
|
+
// -----------------------------------------------------------------------
|
|
92
|
+
async request(endpoint, params = {}) {
|
|
93
|
+
await this.rateLimiter.acquire();
|
|
94
|
+
const url = new URL(`${this.baseUrl}${endpoint}`);
|
|
95
|
+
url.searchParams.set("api_key", this.apiKey);
|
|
96
|
+
for (const [key, value] of Object.entries(params)) {
|
|
97
|
+
if (value !== void 0 && value !== "") {
|
|
98
|
+
url.searchParams.set(key, String(value));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const controller = new AbortController();
|
|
102
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
103
|
+
try {
|
|
104
|
+
console.log(`${LOG_PREFIX} GET ${endpoint}`);
|
|
105
|
+
const response = await fetch(url.toString(), {
|
|
106
|
+
headers: { Accept: "application/json" },
|
|
107
|
+
signal: controller.signal
|
|
108
|
+
});
|
|
109
|
+
clearTimeout(timeoutId);
|
|
110
|
+
if (!response.ok) {
|
|
111
|
+
const body = await response.text().catch(() => "");
|
|
112
|
+
throw new FECClientError(
|
|
113
|
+
`FEC API error ${response.status}: ${response.statusText} \u2014 ${body}`,
|
|
114
|
+
response.status
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
return await response.json();
|
|
118
|
+
} catch (error) {
|
|
119
|
+
clearTimeout(timeoutId);
|
|
120
|
+
if (error instanceof FECClientError) throw error;
|
|
121
|
+
if (error.name === "AbortError") {
|
|
122
|
+
throw new FECClientError("FEC request timed out");
|
|
123
|
+
}
|
|
124
|
+
throw new FECClientError(
|
|
125
|
+
`FEC network error: ${error instanceof Error ? error.message : String(error)}`
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Automatically paginate through all result pages up to maxPages.
|
|
131
|
+
*/
|
|
132
|
+
async paginate(endpoint, params, extractItems) {
|
|
133
|
+
const items = [];
|
|
134
|
+
let page = 1;
|
|
135
|
+
while (page <= this.maxPages) {
|
|
136
|
+
const body = await this.request(endpoint, {
|
|
137
|
+
...params,
|
|
138
|
+
page: String(page),
|
|
139
|
+
per_page: "100"
|
|
140
|
+
});
|
|
141
|
+
const pageItems = extractItems(body);
|
|
142
|
+
items.push(...pageItems);
|
|
143
|
+
const pagination = body.pagination;
|
|
144
|
+
if (!pagination || page >= (pagination.pages ?? 1) || pageItems.length === 0) {
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
page++;
|
|
148
|
+
}
|
|
149
|
+
return items;
|
|
150
|
+
}
|
|
151
|
+
// -----------------------------------------------------------------------
|
|
152
|
+
// Candidates
|
|
153
|
+
// -----------------------------------------------------------------------
|
|
154
|
+
async searchCandidates(query, opts = {}) {
|
|
155
|
+
return this.paginate(
|
|
156
|
+
"/candidates/search/",
|
|
157
|
+
{
|
|
158
|
+
q: query,
|
|
159
|
+
office: opts.office,
|
|
160
|
+
state: opts.state,
|
|
161
|
+
cycle: opts.cycle,
|
|
162
|
+
sort: "name"
|
|
163
|
+
},
|
|
164
|
+
(body) => (body.results ?? []).map((r) => this.mapCandidate(r))
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
async getCandidate(candidateId) {
|
|
168
|
+
const body = await this.request(`/candidate/${candidateId}/`);
|
|
169
|
+
const results = body.results ?? [];
|
|
170
|
+
if (results.length === 0) {
|
|
171
|
+
throw new FECClientError(`Candidate ${candidateId} not found`, 404);
|
|
172
|
+
}
|
|
173
|
+
return this.mapCandidate(results[0]);
|
|
174
|
+
}
|
|
175
|
+
mapCandidate(r) {
|
|
176
|
+
return {
|
|
177
|
+
candidateId: r.candidate_id ?? "",
|
|
178
|
+
name: r.name ?? "",
|
|
179
|
+
party: r.party ?? "",
|
|
180
|
+
office: r.office ?? "",
|
|
181
|
+
state: r.state ?? "",
|
|
182
|
+
district: r.district ?? "",
|
|
183
|
+
incumbentChallenger: r.incumbent_challenge ?? "",
|
|
184
|
+
cycles: r.cycles ?? [],
|
|
185
|
+
activeThrough: r.active_through ?? null
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
// -----------------------------------------------------------------------
|
|
189
|
+
// Committees
|
|
190
|
+
// -----------------------------------------------------------------------
|
|
191
|
+
async searchCommittees(query) {
|
|
192
|
+
return this.paginate(
|
|
193
|
+
"/committees/",
|
|
194
|
+
{ q: query, sort: "name" },
|
|
195
|
+
(body) => (body.results ?? []).map((r) => this.mapCommittee(r))
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
async getCommittee(committeeId) {
|
|
199
|
+
const body = await this.request(`/committee/${committeeId}/`);
|
|
200
|
+
const results = body.results ?? [];
|
|
201
|
+
if (results.length === 0) {
|
|
202
|
+
throw new FECClientError(`Committee ${committeeId} not found`, 404);
|
|
203
|
+
}
|
|
204
|
+
return this.mapCommittee(results[0]);
|
|
205
|
+
}
|
|
206
|
+
mapCommittee(r) {
|
|
207
|
+
return {
|
|
208
|
+
committeeId: r.committee_id ?? "",
|
|
209
|
+
name: r.name ?? "",
|
|
210
|
+
designation: r.designation ?? "",
|
|
211
|
+
type: r.committee_type ?? "",
|
|
212
|
+
party: r.party ?? "",
|
|
213
|
+
state: r.state ?? "",
|
|
214
|
+
treasurerName: r.treasurer_name ?? "",
|
|
215
|
+
candidateIds: r.candidate_ids ?? [],
|
|
216
|
+
cycles: r.cycles ?? []
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
// -----------------------------------------------------------------------
|
|
220
|
+
// Contributions (Schedule A)
|
|
221
|
+
// -----------------------------------------------------------------------
|
|
222
|
+
async getContributions(opts) {
|
|
223
|
+
const params = {
|
|
224
|
+
committee_id: opts.committeeId,
|
|
225
|
+
candidate_id: opts.candidateId,
|
|
226
|
+
contributor_name: opts.contributorName,
|
|
227
|
+
min_amount: opts.minAmount,
|
|
228
|
+
max_amount: opts.maxAmount,
|
|
229
|
+
two_year_transaction_period: opts.cycle,
|
|
230
|
+
sort: "-contribution_receipt_date"
|
|
231
|
+
};
|
|
232
|
+
return this.paginate(
|
|
233
|
+
"/schedules/schedule_a/",
|
|
234
|
+
params,
|
|
235
|
+
(body) => (body.results ?? []).map((r) => this.mapContribution(r))
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
async getDonorLookup(name, state) {
|
|
239
|
+
return this.getContributions({
|
|
240
|
+
contributorName: name,
|
|
241
|
+
...state ? {} : {}
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
mapContribution(r) {
|
|
245
|
+
return {
|
|
246
|
+
committeeId: r.committee_id ?? "",
|
|
247
|
+
committeeName: r.committee?.name ?? r.committee_name ?? "",
|
|
248
|
+
contributorName: r.contributor_name ?? "",
|
|
249
|
+
contributorCity: r.contributor_city ?? "",
|
|
250
|
+
contributorState: r.contributor_state ?? "",
|
|
251
|
+
contributorZip: r.contributor_zip ?? "",
|
|
252
|
+
contributorEmployer: r.contributor_employer ?? "",
|
|
253
|
+
contributorOccupation: r.contributor_occupation ?? "",
|
|
254
|
+
amount: r.contribution_receipt_amount ?? 0,
|
|
255
|
+
date: r.contribution_receipt_date ?? "",
|
|
256
|
+
receiptType: r.receipt_type ?? "",
|
|
257
|
+
memoText: r.memo_text ?? "",
|
|
258
|
+
transactionId: r.transaction_id ?? ""
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
// -----------------------------------------------------------------------
|
|
262
|
+
// Disbursements (Schedule B)
|
|
263
|
+
// -----------------------------------------------------------------------
|
|
264
|
+
async getDisbursements(committeeId, cycle) {
|
|
265
|
+
return this.paginate(
|
|
266
|
+
"/schedules/schedule_b/",
|
|
267
|
+
{
|
|
268
|
+
committee_id: committeeId,
|
|
269
|
+
two_year_transaction_period: cycle,
|
|
270
|
+
sort: "-disbursement_date"
|
|
271
|
+
},
|
|
272
|
+
(body) => (body.results ?? []).map((r) => this.mapDisbursement(r))
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
mapDisbursement(r) {
|
|
276
|
+
return {
|
|
277
|
+
committeeId: r.committee_id ?? "",
|
|
278
|
+
committeeName: r.committee?.name ?? r.committee_name ?? "",
|
|
279
|
+
recipientName: r.recipient_name ?? "",
|
|
280
|
+
recipientCity: r.recipient_city ?? "",
|
|
281
|
+
recipientState: r.recipient_state ?? "",
|
|
282
|
+
amount: r.disbursement_amount ?? 0,
|
|
283
|
+
date: r.disbursement_date ?? "",
|
|
284
|
+
description: r.disbursement_description ?? "",
|
|
285
|
+
categoryCode: r.disbursement_type ?? "",
|
|
286
|
+
memoText: r.memo_text ?? ""
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
// -----------------------------------------------------------------------
|
|
290
|
+
// Filings
|
|
291
|
+
// -----------------------------------------------------------------------
|
|
292
|
+
async getFilings(committeeId) {
|
|
293
|
+
return this.paginate(
|
|
294
|
+
"/committee/${committeeId}/filings/".replace(
|
|
295
|
+
"${committeeId}",
|
|
296
|
+
committeeId
|
|
297
|
+
),
|
|
298
|
+
{ sort: "-receipt_date" },
|
|
299
|
+
(body) => (body.results ?? []).map((r) => this.mapFiling(r))
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
mapFiling(r) {
|
|
303
|
+
return {
|
|
304
|
+
filingId: r.filing_id ?? 0,
|
|
305
|
+
committeeId: r.committee_id ?? "",
|
|
306
|
+
committeeName: r.committee_name ?? "",
|
|
307
|
+
formType: r.form_type ?? "",
|
|
308
|
+
reportType: r.report_type ?? "",
|
|
309
|
+
reportYear: r.report_year ?? 0,
|
|
310
|
+
coverageStartDate: r.coverage_start_date ?? "",
|
|
311
|
+
coverageEndDate: r.coverage_end_date ?? "",
|
|
312
|
+
totalReceipts: r.total_receipts ?? 0,
|
|
313
|
+
totalDisbursements: r.total_disbursements ?? 0,
|
|
314
|
+
cashOnHandEnd: r.cash_on_hand_end_period ?? 0,
|
|
315
|
+
filingDate: r.receipt_date ?? "",
|
|
316
|
+
documentUrl: r.document_url ?? r.pdf_url ?? ""
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
};
|
|
320
|
+
|
|
321
|
+
// src/integrations/public-records/propublica990-client.ts
|
|
322
|
+
var ProPublica990ClientError = class extends Error {
|
|
323
|
+
constructor(message, statusCode) {
|
|
324
|
+
super(message);
|
|
325
|
+
this.statusCode = statusCode;
|
|
326
|
+
this.name = "ProPublica990ClientError";
|
|
327
|
+
}
|
|
328
|
+
};
|
|
329
|
+
var LOG_PREFIX2 = "[OSINT:IRS990]";
|
|
330
|
+
var ProPublica990Client = class {
|
|
331
|
+
baseUrl = "https://projects.propublica.org/nonprofits/api/v2";
|
|
332
|
+
timeout;
|
|
333
|
+
maxPages;
|
|
334
|
+
rateLimiter;
|
|
335
|
+
constructor(config = {}) {
|
|
336
|
+
this.timeout = config.timeout ?? 15e3;
|
|
337
|
+
this.maxPages = config.maxPages ?? 10;
|
|
338
|
+
this.rateLimiter = createRateLimiter("ProPublica990", 5, 1e3);
|
|
339
|
+
}
|
|
340
|
+
// -----------------------------------------------------------------------
|
|
341
|
+
// Internal helpers
|
|
342
|
+
// -----------------------------------------------------------------------
|
|
343
|
+
async request(endpoint, params = {}) {
|
|
344
|
+
await this.rateLimiter.acquire();
|
|
345
|
+
const url = new URL(`${this.baseUrl}${endpoint}`);
|
|
346
|
+
for (const [key, value] of Object.entries(params)) {
|
|
347
|
+
if (value !== void 0 && value !== "") {
|
|
348
|
+
url.searchParams.set(key, String(value));
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
const controller = new AbortController();
|
|
352
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
353
|
+
try {
|
|
354
|
+
console.log(`${LOG_PREFIX2} GET ${endpoint}`);
|
|
355
|
+
const response = await fetch(url.toString(), {
|
|
356
|
+
headers: { Accept: "application/json" },
|
|
357
|
+
signal: controller.signal
|
|
358
|
+
});
|
|
359
|
+
clearTimeout(timeoutId);
|
|
360
|
+
if (!response.ok) {
|
|
361
|
+
const body = await response.text().catch(() => "");
|
|
362
|
+
throw new ProPublica990ClientError(
|
|
363
|
+
`ProPublica API error ${response.status}: ${response.statusText} \u2014 ${body}`,
|
|
364
|
+
response.status
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
return await response.json();
|
|
368
|
+
} catch (error) {
|
|
369
|
+
clearTimeout(timeoutId);
|
|
370
|
+
if (error instanceof ProPublica990ClientError) throw error;
|
|
371
|
+
if (error.name === "AbortError") {
|
|
372
|
+
throw new ProPublica990ClientError("ProPublica request timed out");
|
|
373
|
+
}
|
|
374
|
+
throw new ProPublica990ClientError(
|
|
375
|
+
`ProPublica network error: ${error instanceof Error ? error.message : String(error)}`
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
// -----------------------------------------------------------------------
|
|
380
|
+
// Search organizations
|
|
381
|
+
// -----------------------------------------------------------------------
|
|
382
|
+
async searchOrganizations(query, state) {
|
|
383
|
+
const allOrgs = [];
|
|
384
|
+
let page = 0;
|
|
385
|
+
while (page < this.maxPages) {
|
|
386
|
+
const body = await this.request(
|
|
387
|
+
`/search.json`,
|
|
388
|
+
{
|
|
389
|
+
q: query,
|
|
390
|
+
state,
|
|
391
|
+
page
|
|
392
|
+
}
|
|
393
|
+
);
|
|
394
|
+
const orgs = body.organizations ?? [];
|
|
395
|
+
if (orgs.length === 0) break;
|
|
396
|
+
for (const r of orgs) {
|
|
397
|
+
allOrgs.push(this.mapOrg(r));
|
|
398
|
+
}
|
|
399
|
+
if (orgs.length < 25) break;
|
|
400
|
+
page++;
|
|
401
|
+
}
|
|
402
|
+
return allOrgs;
|
|
403
|
+
}
|
|
404
|
+
// -----------------------------------------------------------------------
|
|
405
|
+
// Get single organization
|
|
406
|
+
// -----------------------------------------------------------------------
|
|
407
|
+
async getOrganization(ein) {
|
|
408
|
+
const normalizedEin = ein.replace(/-/g, "");
|
|
409
|
+
const body = await this.request(
|
|
410
|
+
`/organizations/${normalizedEin}.json`
|
|
411
|
+
);
|
|
412
|
+
const org = body.organization ?? {};
|
|
413
|
+
const filings = (body.filings_with_data ?? []).map(
|
|
414
|
+
(f) => this.mapFiling(f)
|
|
415
|
+
);
|
|
416
|
+
return {
|
|
417
|
+
ein: org.ein ? String(org.ein) : normalizedEin,
|
|
418
|
+
name: org.name ?? "",
|
|
419
|
+
city: org.city ?? "",
|
|
420
|
+
state: org.state ?? "",
|
|
421
|
+
nteeCode: org.ntee_code ?? "",
|
|
422
|
+
subsectionCode: org.subsection_code ?? null,
|
|
423
|
+
classificationCodes: org.classification_codes ?? "",
|
|
424
|
+
rulingDate: org.ruling_date ?? "",
|
|
425
|
+
filings
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
// -----------------------------------------------------------------------
|
|
429
|
+
// Get filings for an EIN
|
|
430
|
+
// -----------------------------------------------------------------------
|
|
431
|
+
async getFilings(ein) {
|
|
432
|
+
const detail = await this.getOrganization(ein);
|
|
433
|
+
return detail.filings;
|
|
434
|
+
}
|
|
435
|
+
// -----------------------------------------------------------------------
|
|
436
|
+
// Mappers
|
|
437
|
+
// -----------------------------------------------------------------------
|
|
438
|
+
mapOrg(r) {
|
|
439
|
+
return {
|
|
440
|
+
ein: r.ein ? String(r.ein) : "",
|
|
441
|
+
name: r.name ?? "",
|
|
442
|
+
city: r.city ?? "",
|
|
443
|
+
state: r.state ?? "",
|
|
444
|
+
nteeCode: r.ntee_code ?? "",
|
|
445
|
+
subsectionCode: r.subsection_code ?? null,
|
|
446
|
+
classificationCodes: r.classification_codes ?? "",
|
|
447
|
+
rulingDate: r.ruling_date ?? "",
|
|
448
|
+
score: r.score ?? 0
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
mapFiling(f) {
|
|
452
|
+
return {
|
|
453
|
+
taxPeriod: f.tax_prd ? String(f.tax_prd) : "",
|
|
454
|
+
taxPeriodBegin: f.tax_prd_yr ? `${f.tax_prd_yr}-01-01` : "",
|
|
455
|
+
taxPeriodEnd: f.tax_prd ? String(f.tax_prd) : "",
|
|
456
|
+
formType: f.formtype ?? f.form_type ?? "",
|
|
457
|
+
pdfUrl: f.pdf_url ?? "",
|
|
458
|
+
updatedAt: f.updated ?? "",
|
|
459
|
+
totalRevenue: f.totrevenue ?? 0,
|
|
460
|
+
totalExpenses: f.totfuncexpns ?? 0,
|
|
461
|
+
totalAssets: f.totassetsend ?? 0,
|
|
462
|
+
totalLiabilities: f.totliabend ?? 0
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
|
|
467
|
+
// src/integrations/public-records/usaspending-client.ts
|
|
468
|
+
var USASpendingClientError = class extends Error {
|
|
469
|
+
constructor(message, statusCode) {
|
|
470
|
+
super(message);
|
|
471
|
+
this.statusCode = statusCode;
|
|
472
|
+
this.name = "USASpendingClientError";
|
|
473
|
+
}
|
|
474
|
+
};
|
|
475
|
+
var LOG_PREFIX3 = "[OSINT:USASpending]";
|
|
476
|
+
var USASpendingClient = class {
|
|
477
|
+
baseUrl = "https://api.usaspending.gov/api/v2";
|
|
478
|
+
timeout;
|
|
479
|
+
maxPages;
|
|
480
|
+
rateLimiter;
|
|
481
|
+
constructor(config = {}) {
|
|
482
|
+
this.timeout = config.timeout ?? 2e4;
|
|
483
|
+
this.maxPages = config.maxPages ?? 10;
|
|
484
|
+
this.rateLimiter = createRateLimiter("USASpending", 10, 1e3);
|
|
485
|
+
}
|
|
486
|
+
// -----------------------------------------------------------------------
|
|
487
|
+
// Internal helpers
|
|
488
|
+
// -----------------------------------------------------------------------
|
|
489
|
+
async post(endpoint, body) {
|
|
490
|
+
await this.rateLimiter.acquire();
|
|
491
|
+
const url = `${this.baseUrl}${endpoint}`;
|
|
492
|
+
const controller = new AbortController();
|
|
493
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
494
|
+
try {
|
|
495
|
+
console.log(`${LOG_PREFIX3} POST ${endpoint}`);
|
|
496
|
+
const response = await fetch(url, {
|
|
497
|
+
method: "POST",
|
|
498
|
+
headers: {
|
|
499
|
+
"Content-Type": "application/json",
|
|
500
|
+
Accept: "application/json"
|
|
501
|
+
},
|
|
502
|
+
body: JSON.stringify(body),
|
|
503
|
+
signal: controller.signal
|
|
504
|
+
});
|
|
505
|
+
clearTimeout(timeoutId);
|
|
506
|
+
if (!response.ok) {
|
|
507
|
+
const text = await response.text().catch(() => "");
|
|
508
|
+
throw new USASpendingClientError(
|
|
509
|
+
`USAspending API error ${response.status}: ${response.statusText} \u2014 ${text}`,
|
|
510
|
+
response.status
|
|
511
|
+
);
|
|
512
|
+
}
|
|
513
|
+
return await response.json();
|
|
514
|
+
} catch (error) {
|
|
515
|
+
clearTimeout(timeoutId);
|
|
516
|
+
if (error instanceof USASpendingClientError) throw error;
|
|
517
|
+
if (error.name === "AbortError") {
|
|
518
|
+
throw new USASpendingClientError("USAspending request timed out");
|
|
519
|
+
}
|
|
520
|
+
throw new USASpendingClientError(
|
|
521
|
+
`USAspending network error: ${error instanceof Error ? error.message : String(error)}`
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
async get(endpoint, params = {}) {
|
|
526
|
+
await this.rateLimiter.acquire();
|
|
527
|
+
const url = new URL(`${this.baseUrl}${endpoint}`);
|
|
528
|
+
for (const [key, value] of Object.entries(params)) {
|
|
529
|
+
if (value !== void 0 && value !== "") {
|
|
530
|
+
url.searchParams.set(key, String(value));
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
const controller = new AbortController();
|
|
534
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
535
|
+
try {
|
|
536
|
+
console.log(`${LOG_PREFIX3} GET ${endpoint}`);
|
|
537
|
+
const response = await fetch(url.toString(), {
|
|
538
|
+
headers: { Accept: "application/json" },
|
|
539
|
+
signal: controller.signal
|
|
540
|
+
});
|
|
541
|
+
clearTimeout(timeoutId);
|
|
542
|
+
if (!response.ok) {
|
|
543
|
+
const text = await response.text().catch(() => "");
|
|
544
|
+
throw new USASpendingClientError(
|
|
545
|
+
`USAspending API error ${response.status}: ${response.statusText} \u2014 ${text}`,
|
|
546
|
+
response.status
|
|
547
|
+
);
|
|
548
|
+
}
|
|
549
|
+
return await response.json();
|
|
550
|
+
} catch (error) {
|
|
551
|
+
clearTimeout(timeoutId);
|
|
552
|
+
if (error instanceof USASpendingClientError) throw error;
|
|
553
|
+
if (error.name === "AbortError") {
|
|
554
|
+
throw new USASpendingClientError("USAspending request timed out");
|
|
555
|
+
}
|
|
556
|
+
throw new USASpendingClientError(
|
|
557
|
+
`USAspending network error: ${error instanceof Error ? error.message : String(error)}`
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
// -----------------------------------------------------------------------
|
|
562
|
+
// Awards search
|
|
563
|
+
// -----------------------------------------------------------------------
|
|
564
|
+
async searchAwards(filters = {}) {
|
|
565
|
+
const allAwards = [];
|
|
566
|
+
let page = 1;
|
|
567
|
+
while (page <= this.maxPages) {
|
|
568
|
+
const apiFilters = {};
|
|
569
|
+
if (filters.keyword) {
|
|
570
|
+
apiFilters.keywords = [filters.keyword];
|
|
571
|
+
}
|
|
572
|
+
if (filters.recipient) {
|
|
573
|
+
apiFilters.recipient_search_text = [filters.recipient];
|
|
574
|
+
}
|
|
575
|
+
if (filters.agency) {
|
|
576
|
+
apiFilters.agencies = [
|
|
577
|
+
{
|
|
578
|
+
type: "awarding",
|
|
579
|
+
tier: "toptier",
|
|
580
|
+
name: filters.agency
|
|
581
|
+
}
|
|
582
|
+
];
|
|
583
|
+
}
|
|
584
|
+
if (filters.dateRange) {
|
|
585
|
+
apiFilters.time_period = [
|
|
586
|
+
{
|
|
587
|
+
start_date: filters.dateRange.start,
|
|
588
|
+
end_date: filters.dateRange.end
|
|
589
|
+
}
|
|
590
|
+
];
|
|
591
|
+
}
|
|
592
|
+
if (filters.awardType && filters.awardType.length > 0) {
|
|
593
|
+
apiFilters.award_type_codes = filters.awardType;
|
|
594
|
+
}
|
|
595
|
+
const body = await this.post("/search/spending_by_award/", {
|
|
596
|
+
filters: apiFilters,
|
|
597
|
+
fields: [
|
|
598
|
+
"Award ID",
|
|
599
|
+
"Description",
|
|
600
|
+
"Award Amount",
|
|
601
|
+
"Total Outlays",
|
|
602
|
+
"Start Date",
|
|
603
|
+
"End Date",
|
|
604
|
+
"Recipient Name",
|
|
605
|
+
"Awarding Agency",
|
|
606
|
+
"Awarding Sub Agency",
|
|
607
|
+
"Funding Agency",
|
|
608
|
+
"Place of Performance City Code",
|
|
609
|
+
"Place of Performance State Code",
|
|
610
|
+
"generated_unique_award_id",
|
|
611
|
+
"recipient_id",
|
|
612
|
+
"Award Type"
|
|
613
|
+
],
|
|
614
|
+
page,
|
|
615
|
+
limit: 100,
|
|
616
|
+
sort: "Award Amount",
|
|
617
|
+
order: "desc"
|
|
618
|
+
});
|
|
619
|
+
const results = body.results ?? [];
|
|
620
|
+
if (results.length === 0) break;
|
|
621
|
+
for (const r of results) {
|
|
622
|
+
allAwards.push({
|
|
623
|
+
awardId: r["Award ID"] ?? "",
|
|
624
|
+
generatedUniqueAwardId: r.generated_unique_award_id ?? "",
|
|
625
|
+
type: r["Award Type"] ?? "",
|
|
626
|
+
typeDescription: r["Award Type"] ?? "",
|
|
627
|
+
description: r["Description"] ?? "",
|
|
628
|
+
totalObligationAmount: r["Award Amount"] ?? 0,
|
|
629
|
+
totalOutlayAmount: r["Total Outlays"] ?? 0,
|
|
630
|
+
dateOfAward: r["Start Date"] ?? "",
|
|
631
|
+
startDate: r["Start Date"] ?? "",
|
|
632
|
+
endDate: r["End Date"] ?? "",
|
|
633
|
+
recipientName: r["Recipient Name"] ?? "",
|
|
634
|
+
recipientUei: r.recipient_id ?? "",
|
|
635
|
+
awardingAgencyName: r["Awarding Agency"] ?? "",
|
|
636
|
+
awardingSubAgencyName: r["Awarding Sub Agency"] ?? "",
|
|
637
|
+
fundingAgencyName: r["Funding Agency"] ?? "",
|
|
638
|
+
placeOfPerformanceCity: r["Place of Performance City Code"] ?? "",
|
|
639
|
+
placeOfPerformanceState: r["Place of Performance State Code"] ?? ""
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
if (!body.page_metadata || page >= (body.page_metadata.num_pages ?? 1)) {
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
page++;
|
|
646
|
+
}
|
|
647
|
+
return allAwards;
|
|
648
|
+
}
|
|
649
|
+
// -----------------------------------------------------------------------
|
|
650
|
+
// Recipients
|
|
651
|
+
// -----------------------------------------------------------------------
|
|
652
|
+
async searchRecipients(query) {
|
|
653
|
+
const allRecipients = [];
|
|
654
|
+
let page = 1;
|
|
655
|
+
while (page <= this.maxPages) {
|
|
656
|
+
const body = await this.post("/recipient/duns/", {
|
|
657
|
+
keyword: query,
|
|
658
|
+
page,
|
|
659
|
+
limit: 100
|
|
660
|
+
});
|
|
661
|
+
const results = body.results ?? [];
|
|
662
|
+
if (results.length === 0) break;
|
|
663
|
+
for (const r of results) {
|
|
664
|
+
allRecipients.push(this.mapRecipient(r));
|
|
665
|
+
}
|
|
666
|
+
if (results.length < 100) break;
|
|
667
|
+
page++;
|
|
668
|
+
}
|
|
669
|
+
return allRecipients;
|
|
670
|
+
}
|
|
671
|
+
async getRecipient(recipientId) {
|
|
672
|
+
const body = await this.get(
|
|
673
|
+
`/recipient/${encodeURIComponent(recipientId)}/`
|
|
674
|
+
);
|
|
675
|
+
return this.mapRecipient(body);
|
|
676
|
+
}
|
|
677
|
+
mapRecipient(r) {
|
|
678
|
+
return {
|
|
679
|
+
recipientId: r.recipient_id ?? r.id ?? "",
|
|
680
|
+
name: r.name ?? r.recipient_name ?? "",
|
|
681
|
+
uei: r.uei ?? "",
|
|
682
|
+
duns: r.duns ?? "",
|
|
683
|
+
recipientLevel: r.recipient_level ?? "",
|
|
684
|
+
totalTransactionAmount: r.total_transaction_amount ?? 0,
|
|
685
|
+
totalFaceValueOfLoans: r.total_face_value_of_loans ?? 0,
|
|
686
|
+
totalContractAmount: r.total_contract_amount ?? 0,
|
|
687
|
+
totalGrantAmount: r.total_grant_amount ?? 0,
|
|
688
|
+
city: r.location?.city_name ?? r.city ?? "",
|
|
689
|
+
state: r.location?.state_code ?? r.state ?? "",
|
|
690
|
+
congressionalDistrict: r.location?.congressional_code ?? "",
|
|
691
|
+
businessTypes: r.business_types ?? []
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
// -----------------------------------------------------------------------
|
|
695
|
+
// Agency spending
|
|
696
|
+
// -----------------------------------------------------------------------
|
|
697
|
+
async getAgencySpending(agencyCode, fiscalYear) {
|
|
698
|
+
const fy = fiscalYear ?? (/* @__PURE__ */ new Date()).getFullYear();
|
|
699
|
+
const body = await this.get(
|
|
700
|
+
`/agency/${agencyCode}/`,
|
|
701
|
+
{ fiscal_year: fy }
|
|
702
|
+
);
|
|
703
|
+
return {
|
|
704
|
+
agencyCode: body.toptier_code ?? agencyCode,
|
|
705
|
+
agencyName: body.name ?? "",
|
|
706
|
+
fiscalYear: fy,
|
|
707
|
+
totalBudgetaryResources: body.budget_authority_amount ?? 0,
|
|
708
|
+
totalObligations: body.obligated_amount ?? 0,
|
|
709
|
+
totalOutlays: body.outlay_amount ?? 0,
|
|
710
|
+
congressionalJustificationUrl: body.congressional_justification_url ?? ""
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
};
|
|
714
|
+
|
|
715
|
+
// src/integrations/public-records/sec-edgar-client.ts
|
|
716
|
+
var SECEdgarClientError = class extends Error {
|
|
717
|
+
constructor(message, statusCode) {
|
|
718
|
+
super(message);
|
|
719
|
+
this.statusCode = statusCode;
|
|
720
|
+
this.name = "SECEdgarClientError";
|
|
721
|
+
}
|
|
722
|
+
};
|
|
723
|
+
var LOG_PREFIX4 = "[OSINT:SEC]";
|
|
724
|
+
var SECEdgarClient = class {
|
|
725
|
+
dataBaseUrl = "https://data.sec.gov";
|
|
726
|
+
searchBaseUrl = "https://efts.sec.gov/LATEST";
|
|
727
|
+
userAgent;
|
|
728
|
+
timeout;
|
|
729
|
+
maxPages;
|
|
730
|
+
rateLimiter;
|
|
731
|
+
/** Cache ticker -> CIK lookups so we don't repeat the same search */
|
|
732
|
+
tickerCikCache = /* @__PURE__ */ new Map();
|
|
733
|
+
constructor(config = {}) {
|
|
734
|
+
this.userAgent = config.userAgent ?? env.SEC_EDGAR_USER_AGENT ?? "OpenSentinel/2.1 (contact@opensentinel.ai)";
|
|
735
|
+
this.timeout = config.timeout ?? 15e3;
|
|
736
|
+
this.maxPages = config.maxPages ?? 10;
|
|
737
|
+
this.rateLimiter = createRateLimiter("SEC", 10, 1e3);
|
|
738
|
+
}
|
|
739
|
+
// -----------------------------------------------------------------------
|
|
740
|
+
// Internal helpers
|
|
741
|
+
// -----------------------------------------------------------------------
|
|
742
|
+
/**
|
|
743
|
+
* Pad a CIK number to 10 digits with leading zeros.
|
|
744
|
+
*/
|
|
745
|
+
padCik(cik) {
|
|
746
|
+
return String(cik).replace(/\D/g, "").padStart(10, "0");
|
|
747
|
+
}
|
|
748
|
+
async fetchJson(url) {
|
|
749
|
+
await this.rateLimiter.acquire();
|
|
750
|
+
const controller = new AbortController();
|
|
751
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
752
|
+
try {
|
|
753
|
+
console.log(`${LOG_PREFIX4} GET ${url}`);
|
|
754
|
+
const response = await fetch(url, {
|
|
755
|
+
headers: {
|
|
756
|
+
"User-Agent": this.userAgent,
|
|
757
|
+
Accept: "application/json"
|
|
758
|
+
},
|
|
759
|
+
signal: controller.signal
|
|
760
|
+
});
|
|
761
|
+
clearTimeout(timeoutId);
|
|
762
|
+
if (!response.ok) {
|
|
763
|
+
const body = await response.text().catch(() => "");
|
|
764
|
+
throw new SECEdgarClientError(
|
|
765
|
+
`SEC EDGAR error ${response.status}: ${response.statusText} \u2014 ${body}`,
|
|
766
|
+
response.status
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
return await response.json();
|
|
770
|
+
} catch (error) {
|
|
771
|
+
clearTimeout(timeoutId);
|
|
772
|
+
if (error instanceof SECEdgarClientError) throw error;
|
|
773
|
+
if (error.name === "AbortError") {
|
|
774
|
+
throw new SECEdgarClientError("SEC EDGAR request timed out");
|
|
775
|
+
}
|
|
776
|
+
throw new SECEdgarClientError(
|
|
777
|
+
`SEC EDGAR network error: ${error instanceof Error ? error.message : String(error)}`
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
// -----------------------------------------------------------------------
|
|
782
|
+
// Search companies (full-text search endpoint)
|
|
783
|
+
// -----------------------------------------------------------------------
|
|
784
|
+
async searchCompanies(query) {
|
|
785
|
+
const url = new URL(`${this.searchBaseUrl}/search-index`);
|
|
786
|
+
url.searchParams.set("q", query);
|
|
787
|
+
url.searchParams.set("dateRange", "custom");
|
|
788
|
+
url.searchParams.set("forms", "10-K");
|
|
789
|
+
const searchUrl = new URL(`${this.searchBaseUrl}/search-index`);
|
|
790
|
+
searchUrl.searchParams.set("q", `"${query}"`);
|
|
791
|
+
searchUrl.searchParams.set("forms", "10-K");
|
|
792
|
+
try {
|
|
793
|
+
const tickers = await this.fetchJson(`${this.dataBaseUrl}/files/company_tickers.json`);
|
|
794
|
+
const lowerQuery = query.toLowerCase();
|
|
795
|
+
const matches = [];
|
|
796
|
+
for (const entry of Object.values(tickers)) {
|
|
797
|
+
if (entry.title.toLowerCase().includes(lowerQuery) || entry.ticker.toLowerCase().includes(lowerQuery)) {
|
|
798
|
+
matches.push({
|
|
799
|
+
cik: this.padCik(entry.cik_str),
|
|
800
|
+
name: entry.title,
|
|
801
|
+
ticker: entry.ticker,
|
|
802
|
+
exchange: "",
|
|
803
|
+
sic: "",
|
|
804
|
+
sicDescription: "",
|
|
805
|
+
stateOfIncorporation: "",
|
|
806
|
+
fiscalYearEnd: ""
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
if (matches.length >= 25) break;
|
|
810
|
+
}
|
|
811
|
+
return matches;
|
|
812
|
+
} catch (error) {
|
|
813
|
+
console.log(`${LOG_PREFIX4} Ticker file search failed, falling back to EFTS`);
|
|
814
|
+
const eftsBody = await this.fetchJson(
|
|
815
|
+
`${this.searchBaseUrl}/search-index?q=${encodeURIComponent(query)}&forms=10-K`
|
|
816
|
+
);
|
|
817
|
+
const hits = eftsBody.hits?.hits ?? [];
|
|
818
|
+
return hits.slice(0, 25).map((h) => ({
|
|
819
|
+
cik: this.padCik(h._source?.entity_id ?? ""),
|
|
820
|
+
name: h._source?.entity_name ?? "",
|
|
821
|
+
ticker: "",
|
|
822
|
+
exchange: "",
|
|
823
|
+
sic: "",
|
|
824
|
+
sicDescription: "",
|
|
825
|
+
stateOfIncorporation: "",
|
|
826
|
+
fiscalYearEnd: ""
|
|
827
|
+
}));
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
// -----------------------------------------------------------------------
|
|
831
|
+
// Company filings
|
|
832
|
+
// -----------------------------------------------------------------------
|
|
833
|
+
async getCompanyFilings(cik, formType) {
|
|
834
|
+
const paddedCik = this.padCik(cik);
|
|
835
|
+
const body = await this.fetchJson(
|
|
836
|
+
`${this.dataBaseUrl}/submissions/CIK${paddedCik}.json`
|
|
837
|
+
);
|
|
838
|
+
const recent = body.filings?.recent ?? {};
|
|
839
|
+
const accessionNumbers = recent.accessionNumber ?? [];
|
|
840
|
+
const forms = recent.form ?? [];
|
|
841
|
+
const filingDates = recent.filingDate ?? [];
|
|
842
|
+
const reportDates = recent.reportDate ?? [];
|
|
843
|
+
const acceptanceDateTimes = recent.acceptanceDateTime ?? [];
|
|
844
|
+
const primaryDocuments = recent.primaryDocument ?? [];
|
|
845
|
+
const primaryDocDescriptions = recent.primaryDocDescription ?? [];
|
|
846
|
+
const sizes = recent.size ?? [];
|
|
847
|
+
const allFilings = [];
|
|
848
|
+
for (let i = 0; i < accessionNumbers.length; i++) {
|
|
849
|
+
const form = forms[i] ?? "";
|
|
850
|
+
if (formType && form !== formType) continue;
|
|
851
|
+
const accession = accessionNumbers[i] ?? "";
|
|
852
|
+
const accessionClean = accession.replace(/-/g, "");
|
|
853
|
+
allFilings.push({
|
|
854
|
+
accessionNumber: accession,
|
|
855
|
+
formType: form,
|
|
856
|
+
filingDate: filingDates[i] ?? "",
|
|
857
|
+
reportDate: reportDates[i] ?? "",
|
|
858
|
+
acceptanceDateTime: acceptanceDateTimes[i] ?? "",
|
|
859
|
+
primaryDocument: primaryDocuments[i] ?? "",
|
|
860
|
+
primaryDocDescription: primaryDocDescriptions[i] ?? "",
|
|
861
|
+
filingUrl: `https://www.sec.gov/Archives/edgar/data/${parseInt(cik, 10)}/${accessionClean}/${primaryDocuments[i] ?? ""}`,
|
|
862
|
+
size: sizes[i] ?? 0
|
|
863
|
+
});
|
|
864
|
+
}
|
|
865
|
+
const additionalFiles = body.filings?.files ?? [];
|
|
866
|
+
let pagesLoaded = 1;
|
|
867
|
+
for (const file of additionalFiles) {
|
|
868
|
+
if (pagesLoaded >= this.maxPages) break;
|
|
869
|
+
try {
|
|
870
|
+
const additionalBody = await this.fetchJson(
|
|
871
|
+
`${this.dataBaseUrl}/submissions/${file}`
|
|
872
|
+
);
|
|
873
|
+
const addAccessions = additionalBody.accessionNumber ?? [];
|
|
874
|
+
const addForms = additionalBody.form ?? [];
|
|
875
|
+
const addFilingDates = additionalBody.filingDate ?? [];
|
|
876
|
+
const addReportDates = additionalBody.reportDate ?? [];
|
|
877
|
+
const addAcceptanceDates = additionalBody.acceptanceDateTime ?? [];
|
|
878
|
+
const addPrimaryDocs = additionalBody.primaryDocument ?? [];
|
|
879
|
+
const addPrimaryDescriptions = additionalBody.primaryDocDescription ?? [];
|
|
880
|
+
const addSizes = additionalBody.size ?? [];
|
|
881
|
+
for (let i = 0; i < addAccessions.length; i++) {
|
|
882
|
+
const form = addForms[i] ?? "";
|
|
883
|
+
if (formType && form !== formType) continue;
|
|
884
|
+
const accession = addAccessions[i] ?? "";
|
|
885
|
+
const accessionClean = accession.replace(/-/g, "");
|
|
886
|
+
allFilings.push({
|
|
887
|
+
accessionNumber: accession,
|
|
888
|
+
formType: form,
|
|
889
|
+
filingDate: addFilingDates[i] ?? "",
|
|
890
|
+
reportDate: addReportDates[i] ?? "",
|
|
891
|
+
acceptanceDateTime: addAcceptanceDates[i] ?? "",
|
|
892
|
+
primaryDocument: addPrimaryDocs[i] ?? "",
|
|
893
|
+
primaryDocDescription: addPrimaryDescriptions[i] ?? "",
|
|
894
|
+
filingUrl: `https://www.sec.gov/Archives/edgar/data/${parseInt(cik, 10)}/${accessionClean}/${addPrimaryDocs[i] ?? ""}`,
|
|
895
|
+
size: addSizes[i] ?? 0
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
pagesLoaded++;
|
|
899
|
+
} catch (error) {
|
|
900
|
+
console.log(
|
|
901
|
+
`${LOG_PREFIX4} Failed to fetch additional filings page ${file}: ${error instanceof Error ? error.message : String(error)}`
|
|
902
|
+
);
|
|
903
|
+
break;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
return allFilings;
|
|
907
|
+
}
|
|
908
|
+
// -----------------------------------------------------------------------
|
|
909
|
+
// Insider transactions (Forms 3, 4, 5)
|
|
910
|
+
// -----------------------------------------------------------------------
|
|
911
|
+
async getInsiderTransactions(cik) {
|
|
912
|
+
const filings = await this.getCompanyFilings(cik, "4");
|
|
913
|
+
const transactions = [];
|
|
914
|
+
const paddedCik = this.padCik(cik);
|
|
915
|
+
try {
|
|
916
|
+
const body = await this.fetchJson(
|
|
917
|
+
`${this.dataBaseUrl}/api/xbrl/companyfacts/CIK${paddedCik}.json`
|
|
918
|
+
);
|
|
919
|
+
for (const filing of filings.slice(0, 100)) {
|
|
920
|
+
transactions.push({
|
|
921
|
+
accessionNumber: filing.accessionNumber,
|
|
922
|
+
filingDate: filing.filingDate,
|
|
923
|
+
reportingOwnerCik: "",
|
|
924
|
+
reportingOwnerName: filing.primaryDocDescription || "See filing",
|
|
925
|
+
isDirector: false,
|
|
926
|
+
isOfficer: false,
|
|
927
|
+
officerTitle: "",
|
|
928
|
+
transactionDate: filing.reportDate || filing.filingDate,
|
|
929
|
+
transactionCode: "",
|
|
930
|
+
transactionShares: 0,
|
|
931
|
+
transactionPricePerShare: 0,
|
|
932
|
+
sharesOwnedFollowing: 0,
|
|
933
|
+
directOrIndirect: "D"
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
} catch {
|
|
937
|
+
for (const filing of filings.slice(0, 100)) {
|
|
938
|
+
transactions.push({
|
|
939
|
+
accessionNumber: filing.accessionNumber,
|
|
940
|
+
filingDate: filing.filingDate,
|
|
941
|
+
reportingOwnerCik: "",
|
|
942
|
+
reportingOwnerName: filing.primaryDocDescription || "See filing",
|
|
943
|
+
isDirector: false,
|
|
944
|
+
isOfficer: false,
|
|
945
|
+
officerTitle: "",
|
|
946
|
+
transactionDate: filing.reportDate || filing.filingDate,
|
|
947
|
+
transactionCode: "",
|
|
948
|
+
transactionShares: 0,
|
|
949
|
+
transactionPricePerShare: 0,
|
|
950
|
+
sharesOwnedFollowing: 0,
|
|
951
|
+
directOrIndirect: "D"
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return transactions;
|
|
956
|
+
}
|
|
957
|
+
// -----------------------------------------------------------------------
|
|
958
|
+
// Company facts (structured XBRL data)
|
|
959
|
+
// -----------------------------------------------------------------------
|
|
960
|
+
async getCompanyFacts(cik) {
|
|
961
|
+
const paddedCik = this.padCik(cik);
|
|
962
|
+
const body = await this.fetchJson(
|
|
963
|
+
`${this.dataBaseUrl}/api/xbrl/companyfacts/CIK${paddedCik}.json`
|
|
964
|
+
);
|
|
965
|
+
return {
|
|
966
|
+
cik: paddedCik,
|
|
967
|
+
entityName: body.entityName ?? "",
|
|
968
|
+
facts: body.facts ?? {}
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
// -----------------------------------------------------------------------
|
|
972
|
+
// Ticker -> CIK lookup
|
|
973
|
+
// -----------------------------------------------------------------------
|
|
974
|
+
async lookupCikByTicker(ticker) {
|
|
975
|
+
const upperTicker = ticker.toUpperCase();
|
|
976
|
+
if (this.tickerCikCache.has(upperTicker)) {
|
|
977
|
+
return this.tickerCikCache.get(upperTicker);
|
|
978
|
+
}
|
|
979
|
+
try {
|
|
980
|
+
const tickers = await this.fetchJson(`${this.dataBaseUrl}/files/company_tickers.json`);
|
|
981
|
+
for (const entry of Object.values(tickers)) {
|
|
982
|
+
if (entry.ticker.toUpperCase() === upperTicker) {
|
|
983
|
+
const paddedCik = this.padCik(entry.cik_str);
|
|
984
|
+
this.tickerCikCache.set(upperTicker, paddedCik);
|
|
985
|
+
return paddedCik;
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
return null;
|
|
989
|
+
} catch (error) {
|
|
990
|
+
console.log(
|
|
991
|
+
`${LOG_PREFIX4} Ticker lookup failed for ${ticker}: ${error instanceof Error ? error.message : String(error)}`
|
|
992
|
+
);
|
|
993
|
+
return null;
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
};
|
|
997
|
+
|
|
998
|
+
// src/integrations/public-records/opencorporates-client.ts
|
|
999
|
+
var OpenCorporatesClientError = class extends Error {
|
|
1000
|
+
constructor(message, statusCode) {
|
|
1001
|
+
super(message);
|
|
1002
|
+
this.statusCode = statusCode;
|
|
1003
|
+
this.name = "OpenCorporatesClientError";
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
var LOG_PREFIX5 = "[OSINT:OpenCorporates]";
|
|
1007
|
+
var OpenCorporatesClient = class {
|
|
1008
|
+
baseUrl = "https://api.opencorporates.com/v0.4";
|
|
1009
|
+
apiToken;
|
|
1010
|
+
timeout;
|
|
1011
|
+
maxPages;
|
|
1012
|
+
rateLimiter;
|
|
1013
|
+
constructor(config = {}) {
|
|
1014
|
+
this.apiToken = config.apiToken ?? env.OPENCORPORATES_API_TOKEN ?? "";
|
|
1015
|
+
this.timeout = config.timeout ?? 15e3;
|
|
1016
|
+
this.maxPages = config.maxPages ?? 10;
|
|
1017
|
+
this.rateLimiter = createRateLimiter("OpenCorporates", 1, 1e3);
|
|
1018
|
+
}
|
|
1019
|
+
// -----------------------------------------------------------------------
|
|
1020
|
+
// Internal helpers
|
|
1021
|
+
// -----------------------------------------------------------------------
|
|
1022
|
+
async request(endpoint, params = {}) {
|
|
1023
|
+
await this.rateLimiter.acquire();
|
|
1024
|
+
const url = new URL(`${this.baseUrl}${endpoint}`);
|
|
1025
|
+
if (this.apiToken) {
|
|
1026
|
+
url.searchParams.set("api_token", this.apiToken);
|
|
1027
|
+
}
|
|
1028
|
+
for (const [key, value] of Object.entries(params)) {
|
|
1029
|
+
if (value !== void 0 && value !== "") {
|
|
1030
|
+
url.searchParams.set(key, String(value));
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
const controller = new AbortController();
|
|
1034
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
1035
|
+
try {
|
|
1036
|
+
console.log(`${LOG_PREFIX5} GET ${endpoint}`);
|
|
1037
|
+
const response = await fetch(url.toString(), {
|
|
1038
|
+
headers: { Accept: "application/json" },
|
|
1039
|
+
signal: controller.signal
|
|
1040
|
+
});
|
|
1041
|
+
clearTimeout(timeoutId);
|
|
1042
|
+
if (!response.ok) {
|
|
1043
|
+
const body = await response.text().catch(() => "");
|
|
1044
|
+
throw new OpenCorporatesClientError(
|
|
1045
|
+
`OpenCorporates API error ${response.status}: ${response.statusText} \u2014 ${body}`,
|
|
1046
|
+
response.status
|
|
1047
|
+
);
|
|
1048
|
+
}
|
|
1049
|
+
return await response.json();
|
|
1050
|
+
} catch (error) {
|
|
1051
|
+
clearTimeout(timeoutId);
|
|
1052
|
+
if (error instanceof OpenCorporatesClientError) throw error;
|
|
1053
|
+
if (error.name === "AbortError") {
|
|
1054
|
+
throw new OpenCorporatesClientError(
|
|
1055
|
+
"OpenCorporates request timed out"
|
|
1056
|
+
);
|
|
1057
|
+
}
|
|
1058
|
+
throw new OpenCorporatesClientError(
|
|
1059
|
+
`OpenCorporates network error: ${error instanceof Error ? error.message : String(error)}`
|
|
1060
|
+
);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
// -----------------------------------------------------------------------
|
|
1064
|
+
// Search companies
|
|
1065
|
+
// -----------------------------------------------------------------------
|
|
1066
|
+
async searchCompanies(query, jurisdiction) {
|
|
1067
|
+
const allCompanies = [];
|
|
1068
|
+
let page = 1;
|
|
1069
|
+
while (page <= this.maxPages) {
|
|
1070
|
+
const params = {
|
|
1071
|
+
q: query,
|
|
1072
|
+
page,
|
|
1073
|
+
per_page: 30
|
|
1074
|
+
};
|
|
1075
|
+
if (jurisdiction) {
|
|
1076
|
+
params.jurisdiction_code = jurisdiction;
|
|
1077
|
+
}
|
|
1078
|
+
const body = await this.request("/companies/search", params);
|
|
1079
|
+
const companies = body.results?.companies ?? [];
|
|
1080
|
+
if (companies.length === 0) break;
|
|
1081
|
+
for (const wrapper of companies) {
|
|
1082
|
+
const c = wrapper.company ?? wrapper;
|
|
1083
|
+
allCompanies.push(this.mapCompany(c));
|
|
1084
|
+
}
|
|
1085
|
+
const totalPages = body.results?.total_pages ?? 1;
|
|
1086
|
+
if (page >= totalPages || companies.length < 30) break;
|
|
1087
|
+
page++;
|
|
1088
|
+
}
|
|
1089
|
+
return allCompanies;
|
|
1090
|
+
}
|
|
1091
|
+
// -----------------------------------------------------------------------
|
|
1092
|
+
// Get single company
|
|
1093
|
+
// -----------------------------------------------------------------------
|
|
1094
|
+
async getCompany(jurisdictionCode, companyNumber) {
|
|
1095
|
+
const body = await this.request(
|
|
1096
|
+
`/companies/${encodeURIComponent(jurisdictionCode)}/${encodeURIComponent(companyNumber)}`
|
|
1097
|
+
);
|
|
1098
|
+
const c = body.results?.company ?? body.company ?? {};
|
|
1099
|
+
return this.mapCompany(c);
|
|
1100
|
+
}
|
|
1101
|
+
// -----------------------------------------------------------------------
|
|
1102
|
+
// Search officers
|
|
1103
|
+
// -----------------------------------------------------------------------
|
|
1104
|
+
async searchOfficers(query, jurisdiction) {
|
|
1105
|
+
const allOfficers = [];
|
|
1106
|
+
let page = 1;
|
|
1107
|
+
while (page <= this.maxPages) {
|
|
1108
|
+
const params = {
|
|
1109
|
+
q: query,
|
|
1110
|
+
page,
|
|
1111
|
+
per_page: 30
|
|
1112
|
+
};
|
|
1113
|
+
if (jurisdiction) {
|
|
1114
|
+
params.jurisdiction_code = jurisdiction;
|
|
1115
|
+
}
|
|
1116
|
+
const body = await this.request("/officers/search", params);
|
|
1117
|
+
const officers = body.results?.officers ?? [];
|
|
1118
|
+
if (officers.length === 0) break;
|
|
1119
|
+
for (const wrapper of officers) {
|
|
1120
|
+
const o = wrapper.officer ?? wrapper;
|
|
1121
|
+
allOfficers.push(this.mapOfficer(o));
|
|
1122
|
+
}
|
|
1123
|
+
const totalPages = body.results?.total_pages ?? 1;
|
|
1124
|
+
if (page >= totalPages || officers.length < 30) break;
|
|
1125
|
+
page++;
|
|
1126
|
+
}
|
|
1127
|
+
return allOfficers;
|
|
1128
|
+
}
|
|
1129
|
+
// -----------------------------------------------------------------------
|
|
1130
|
+
// Get filings for a company
|
|
1131
|
+
// -----------------------------------------------------------------------
|
|
1132
|
+
async getFilings(jurisdictionCode, companyNumber) {
|
|
1133
|
+
const allFilings = [];
|
|
1134
|
+
let page = 1;
|
|
1135
|
+
while (page <= this.maxPages) {
|
|
1136
|
+
const body = await this.request(
|
|
1137
|
+
`/companies/${encodeURIComponent(jurisdictionCode)}/${encodeURIComponent(companyNumber)}/filings`,
|
|
1138
|
+
{ page, per_page: 30 }
|
|
1139
|
+
);
|
|
1140
|
+
const filings = body.results?.filings ?? [];
|
|
1141
|
+
if (filings.length === 0) break;
|
|
1142
|
+
for (const wrapper of filings) {
|
|
1143
|
+
const f = wrapper.filing ?? wrapper;
|
|
1144
|
+
allFilings.push(this.mapFiling(f));
|
|
1145
|
+
}
|
|
1146
|
+
const totalPages = body.results?.total_pages ?? 1;
|
|
1147
|
+
if (page >= totalPages || filings.length < 30) break;
|
|
1148
|
+
page++;
|
|
1149
|
+
}
|
|
1150
|
+
return allFilings;
|
|
1151
|
+
}
|
|
1152
|
+
// -----------------------------------------------------------------------
|
|
1153
|
+
// Mappers
|
|
1154
|
+
// -----------------------------------------------------------------------
|
|
1155
|
+
mapCompany(c) {
|
|
1156
|
+
const officers = (c.officers ?? []).map(
|
|
1157
|
+
(wrapper) => {
|
|
1158
|
+
const o = wrapper.officer ?? wrapper;
|
|
1159
|
+
return this.mapOfficer(o);
|
|
1160
|
+
}
|
|
1161
|
+
);
|
|
1162
|
+
return {
|
|
1163
|
+
companyNumber: c.company_number ?? "",
|
|
1164
|
+
name: c.name ?? "",
|
|
1165
|
+
jurisdictionCode: c.jurisdiction_code ?? "",
|
|
1166
|
+
incorporationDate: c.incorporation_date ?? "",
|
|
1167
|
+
dissolutionDate: c.dissolution_date ?? "",
|
|
1168
|
+
companyType: c.company_type ?? "",
|
|
1169
|
+
registryUrl: c.registry_url ?? "",
|
|
1170
|
+
status: c.current_status ?? c.status ?? "",
|
|
1171
|
+
registeredAddress: c.registered_address_in_full ?? c.registered_address ?? "",
|
|
1172
|
+
agentName: c.agent_name ?? "",
|
|
1173
|
+
agentAddress: c.agent_address ?? "",
|
|
1174
|
+
officers,
|
|
1175
|
+
openCorporatesUrl: c.opencorporates_url ?? "",
|
|
1176
|
+
source: c.source?.publisher ?? "",
|
|
1177
|
+
updatedAt: c.updated_at ?? ""
|
|
1178
|
+
};
|
|
1179
|
+
}
|
|
1180
|
+
mapOfficer(o) {
|
|
1181
|
+
const company = o.company ?? {};
|
|
1182
|
+
return {
|
|
1183
|
+
id: o.id ?? 0,
|
|
1184
|
+
name: o.name ?? "",
|
|
1185
|
+
position: o.position ?? "",
|
|
1186
|
+
startDate: o.start_date ?? "",
|
|
1187
|
+
endDate: o.end_date ?? "",
|
|
1188
|
+
nationality: o.nationality ?? "",
|
|
1189
|
+
occupation: o.occupation ?? "",
|
|
1190
|
+
companyNumber: company.company_number ?? "",
|
|
1191
|
+
companyName: company.name ?? "",
|
|
1192
|
+
jurisdictionCode: company.jurisdiction_code ?? ""
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
mapFiling(f) {
|
|
1196
|
+
return {
|
|
1197
|
+
id: f.id ?? 0,
|
|
1198
|
+
title: f.title ?? "",
|
|
1199
|
+
date: f.date ?? "",
|
|
1200
|
+
description: f.description ?? "",
|
|
1201
|
+
filingType: f.filing_type ?? "",
|
|
1202
|
+
url: f.url ?? "",
|
|
1203
|
+
openCorporatesUrl: f.opencorporates_url ?? ""
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
};
|
|
1207
|
+
|
|
1208
|
+
// src/integrations/public-records/index.ts
|
|
1209
|
+
var PublicRecords = class {
|
|
1210
|
+
fec;
|
|
1211
|
+
irs990;
|
|
1212
|
+
usaspending;
|
|
1213
|
+
sec;
|
|
1214
|
+
opencorporates;
|
|
1215
|
+
constructor(config = {}) {
|
|
1216
|
+
this.fec = new FECClient(config.fec);
|
|
1217
|
+
this.irs990 = new ProPublica990Client(config.irs990);
|
|
1218
|
+
this.usaspending = new USASpendingClient(config.usaspending);
|
|
1219
|
+
this.sec = new SECEdgarClient(config.sec);
|
|
1220
|
+
this.opencorporates = new OpenCorporatesClient(config.opencorporates);
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
function createPublicRecords(config = {}) {
|
|
1224
|
+
return new PublicRecords(config);
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
// src/integrations/neo4j/client.ts
|
|
1228
|
+
import neo4j from "neo4j-driver";
|
|
1229
|
+
var Neo4jClient = class {
|
|
1230
|
+
driver;
|
|
1231
|
+
database;
|
|
1232
|
+
constructor(config) {
|
|
1233
|
+
const uri = config?.uri ?? env.NEO4J_URI;
|
|
1234
|
+
const user = config?.user ?? env.NEO4J_USER;
|
|
1235
|
+
const password = config?.password ?? env.NEO4J_PASSWORD;
|
|
1236
|
+
this.database = config?.database ?? env.NEO4J_DATABASE;
|
|
1237
|
+
try {
|
|
1238
|
+
this.driver = neo4j.driver(uri, neo4j.auth.basic(user, password));
|
|
1239
|
+
console.log("[Neo4j] Driver created for", uri);
|
|
1240
|
+
} catch (err) {
|
|
1241
|
+
console.log("[Neo4j] Failed to create driver:", err);
|
|
1242
|
+
throw err;
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
// -------------------------------------------------------------------------
|
|
1246
|
+
// Public helpers
|
|
1247
|
+
// -------------------------------------------------------------------------
|
|
1248
|
+
/**
|
|
1249
|
+
* Execute a read-only Cypher query and return the result records.
|
|
1250
|
+
*/
|
|
1251
|
+
async runQuery(cypher, params) {
|
|
1252
|
+
const session = this.getSession("READ");
|
|
1253
|
+
try {
|
|
1254
|
+
const result = await session.run(cypher, params ?? {});
|
|
1255
|
+
return result;
|
|
1256
|
+
} catch (err) {
|
|
1257
|
+
console.log("[Neo4j] Read query failed:", err);
|
|
1258
|
+
throw err;
|
|
1259
|
+
} finally {
|
|
1260
|
+
await session.close();
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* Execute a write Cypher query and return the result records.
|
|
1265
|
+
*/
|
|
1266
|
+
async runWrite(cypher, params) {
|
|
1267
|
+
const session = this.getSession("WRITE");
|
|
1268
|
+
try {
|
|
1269
|
+
const result = await session.run(cypher, params ?? {});
|
|
1270
|
+
return result;
|
|
1271
|
+
} catch (err) {
|
|
1272
|
+
console.log("[Neo4j] Write query failed:", err);
|
|
1273
|
+
throw err;
|
|
1274
|
+
} finally {
|
|
1275
|
+
await session.close();
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* Verify that the driver can reach the Neo4j server.
|
|
1280
|
+
*/
|
|
1281
|
+
async verifyConnectivity() {
|
|
1282
|
+
try {
|
|
1283
|
+
await this.driver.verifyConnectivity();
|
|
1284
|
+
console.log("[Neo4j] Connectivity verified");
|
|
1285
|
+
return true;
|
|
1286
|
+
} catch (err) {
|
|
1287
|
+
console.log("[Neo4j] Connectivity check failed:", err);
|
|
1288
|
+
return false;
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
/**
|
|
1292
|
+
* Get a raw session for advanced use-cases (caller is responsible for closing).
|
|
1293
|
+
*/
|
|
1294
|
+
getSession(defaultAccessMode = "READ") {
|
|
1295
|
+
return this.driver.session({
|
|
1296
|
+
database: this.database,
|
|
1297
|
+
defaultAccessMode: defaultAccessMode === "READ" ? neo4j.session.READ : neo4j.session.WRITE
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
/**
|
|
1301
|
+
* Gracefully close the underlying driver.
|
|
1302
|
+
*/
|
|
1303
|
+
async close() {
|
|
1304
|
+
try {
|
|
1305
|
+
await this.driver.close();
|
|
1306
|
+
console.log("[Neo4j] Driver closed");
|
|
1307
|
+
} catch (err) {
|
|
1308
|
+
console.log("[Neo4j] Error closing driver:", err);
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
};
|
|
1312
|
+
var _instance = null;
|
|
1313
|
+
function getNeo4jClient(config) {
|
|
1314
|
+
if (!_instance) {
|
|
1315
|
+
_instance = new Neo4jClient(config);
|
|
1316
|
+
}
|
|
1317
|
+
return _instance;
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
// src/integrations/neo4j/graph-ops.ts
|
|
1321
|
+
import { eq, ilike } from "drizzle-orm";
|
|
1322
|
+
async function findEntitiesByName(name, fuzzy = false) {
|
|
1323
|
+
if (fuzzy) {
|
|
1324
|
+
try {
|
|
1325
|
+
const client = getNeo4jClient();
|
|
1326
|
+
const safeName = name.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, "\\$&");
|
|
1327
|
+
const result = await client.runQuery(
|
|
1328
|
+
`
|
|
1329
|
+
CALL db.index.fulltext.queryNodes("entity_fulltext", $query)
|
|
1330
|
+
YIELD node, score
|
|
1331
|
+
RETURN node.pgId AS pgId,
|
|
1332
|
+
node.type AS type,
|
|
1333
|
+
node.name AS name,
|
|
1334
|
+
node.aliases AS aliases,
|
|
1335
|
+
node.description AS description,
|
|
1336
|
+
node.importance AS importance,
|
|
1337
|
+
node.source AS source,
|
|
1338
|
+
score
|
|
1339
|
+
ORDER BY score DESC
|
|
1340
|
+
LIMIT 25
|
|
1341
|
+
`,
|
|
1342
|
+
{ query: `${safeName}~` }
|
|
1343
|
+
);
|
|
1344
|
+
return result.records.map((r) => ({
|
|
1345
|
+
pgId: r.get("pgId"),
|
|
1346
|
+
type: r.get("type"),
|
|
1347
|
+
name: r.get("name"),
|
|
1348
|
+
aliases: r.get("aliases"),
|
|
1349
|
+
description: r.get("description"),
|
|
1350
|
+
importance: typeof r.get("importance") === "number" ? r.get("importance") : void 0,
|
|
1351
|
+
source: r.get("source")
|
|
1352
|
+
}));
|
|
1353
|
+
} catch (err) {
|
|
1354
|
+
console.log("[Neo4j] Fulltext search failed, falling back to Postgres:", err);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
const rows = await db.select().from(graphEntities).where(ilike(graphEntities.name, `%${name}%`)).limit(25);
|
|
1358
|
+
return rows.map((r) => ({
|
|
1359
|
+
pgId: r.id,
|
|
1360
|
+
type: r.type,
|
|
1361
|
+
name: r.name,
|
|
1362
|
+
aliases: r.aliases ?? [],
|
|
1363
|
+
description: r.description ?? void 0,
|
|
1364
|
+
attributes: r.attributes ?? {},
|
|
1365
|
+
importance: r.importance ?? 50
|
|
1366
|
+
}));
|
|
1367
|
+
}
|
|
1368
|
+
async function createRelationship(sourceId, targetId, type, attrs) {
|
|
1369
|
+
const [row] = await db.insert(graphRelationships).values({
|
|
1370
|
+
sourceEntityId: sourceId,
|
|
1371
|
+
targetEntityId: targetId,
|
|
1372
|
+
type,
|
|
1373
|
+
strength: attrs?.strength ?? 50,
|
|
1374
|
+
bidirectional: attrs?.bidirectional ?? false,
|
|
1375
|
+
context: attrs?.context ?? null,
|
|
1376
|
+
attributes: attrs?.attributes ?? {}
|
|
1377
|
+
}).returning({ id: graphRelationships.id });
|
|
1378
|
+
const pgId = row.id;
|
|
1379
|
+
try {
|
|
1380
|
+
const client = getNeo4jClient();
|
|
1381
|
+
await client.runWrite(
|
|
1382
|
+
`
|
|
1383
|
+
MATCH (a:Entity {pgId: $sourceId}), (b:Entity {pgId: $targetId})
|
|
1384
|
+
CREATE (a)-[r:RELATES_TO {
|
|
1385
|
+
pgId: $pgId,
|
|
1386
|
+
type: $type,
|
|
1387
|
+
strength: $strength,
|
|
1388
|
+
context: $context,
|
|
1389
|
+
source: $source
|
|
1390
|
+
}]->(b)
|
|
1391
|
+
`,
|
|
1392
|
+
{
|
|
1393
|
+
sourceId,
|
|
1394
|
+
targetId,
|
|
1395
|
+
pgId,
|
|
1396
|
+
type,
|
|
1397
|
+
strength: attrs?.strength ?? 50,
|
|
1398
|
+
context: attrs?.context ?? "",
|
|
1399
|
+
source: attrs?.source ?? ""
|
|
1400
|
+
}
|
|
1401
|
+
);
|
|
1402
|
+
} catch (err) {
|
|
1403
|
+
console.log("[Neo4j] Failed to create relationship in Neo4j:", err);
|
|
1404
|
+
}
|
|
1405
|
+
return pgId;
|
|
1406
|
+
}
|
|
1407
|
+
async function getNeighbors(entityPgId, depth = 2) {
|
|
1408
|
+
const client = getNeo4jClient();
|
|
1409
|
+
try {
|
|
1410
|
+
const result = await client.runQuery(
|
|
1411
|
+
`
|
|
1412
|
+
MATCH path = (start:Entity {pgId: $pgId})-[*1..${Math.min(depth, 10)}]-(neighbor:Entity)
|
|
1413
|
+
WITH DISTINCT neighbor, relationships(path) AS rels, nodes(path) AS ns
|
|
1414
|
+
UNWIND rels AS rel
|
|
1415
|
+
WITH COLLECT(DISTINCT {
|
|
1416
|
+
pgId: neighbor.pgId,
|
|
1417
|
+
name: neighbor.name,
|
|
1418
|
+
type: neighbor.type,
|
|
1419
|
+
importance: neighbor.importance
|
|
1420
|
+
}) AS nodeList,
|
|
1421
|
+
COLLECT(DISTINCT {
|
|
1422
|
+
sourcePgId: startNode(rel).pgId,
|
|
1423
|
+
targetPgId: endNode(rel).pgId,
|
|
1424
|
+
type: rel.type,
|
|
1425
|
+
strength: rel.strength
|
|
1426
|
+
}) AS edgeList
|
|
1427
|
+
RETURN nodeList, edgeList
|
|
1428
|
+
`,
|
|
1429
|
+
{ pgId: entityPgId }
|
|
1430
|
+
);
|
|
1431
|
+
if (result.records.length === 0) {
|
|
1432
|
+
return { nodes: [], edges: [] };
|
|
1433
|
+
}
|
|
1434
|
+
const record = result.records[0];
|
|
1435
|
+
const nodes = record.get("nodeList") ?? [];
|
|
1436
|
+
const edges = record.get("edgeList") ?? [];
|
|
1437
|
+
return { nodes, edges };
|
|
1438
|
+
} catch (err) {
|
|
1439
|
+
console.log("[Neo4j] getNeighbors failed:", err);
|
|
1440
|
+
return { nodes: [], edges: [] };
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
async function findShortestPath(sourcePgId, targetPgId) {
|
|
1444
|
+
const client = getNeo4jClient();
|
|
1445
|
+
try {
|
|
1446
|
+
const result = await client.runQuery(
|
|
1447
|
+
`
|
|
1448
|
+
MATCH (a:Entity {pgId: $sourcePgId}), (b:Entity {pgId: $targetPgId}),
|
|
1449
|
+
path = shortestPath((a)-[*..15]-(b))
|
|
1450
|
+
WITH nodes(path) AS ns, relationships(path) AS rels, length(path) AS pathLen
|
|
1451
|
+
RETURN
|
|
1452
|
+
[n IN ns | {pgId: n.pgId, name: n.name, type: n.type, importance: n.importance}] AS nodes,
|
|
1453
|
+
[r IN rels | {sourcePgId: startNode(r).pgId, targetPgId: endNode(r).pgId, type: r.type, strength: r.strength}] AS edges,
|
|
1454
|
+
pathLen
|
|
1455
|
+
`,
|
|
1456
|
+
{ sourcePgId, targetPgId }
|
|
1457
|
+
);
|
|
1458
|
+
if (result.records.length === 0) {
|
|
1459
|
+
return null;
|
|
1460
|
+
}
|
|
1461
|
+
const record = result.records[0];
|
|
1462
|
+
return {
|
|
1463
|
+
nodes: record.get("nodes"),
|
|
1464
|
+
edges: record.get("edges"),
|
|
1465
|
+
length: typeof record.get("pathLen") === "object" ? record.get("pathLen").toNumber() : record.get("pathLen")
|
|
1466
|
+
};
|
|
1467
|
+
} catch (err) {
|
|
1468
|
+
console.log("[Neo4j] findShortestPath failed:", err);
|
|
1469
|
+
return null;
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
async function getCommunities() {
|
|
1473
|
+
const client = getNeo4jClient();
|
|
1474
|
+
try {
|
|
1475
|
+
const result = await client.runQuery(
|
|
1476
|
+
`
|
|
1477
|
+
MATCH (e:Entity)
|
|
1478
|
+
WITH collect(e) AS allNodes
|
|
1479
|
+
UNWIND allNodes AS node
|
|
1480
|
+
MATCH path = (node)-[*0..]-(connected:Entity)
|
|
1481
|
+
WITH node, collect(DISTINCT connected) AS component
|
|
1482
|
+
WITH component, component[0].pgId AS componentId
|
|
1483
|
+
ORDER BY size(component) DESC
|
|
1484
|
+
WITH DISTINCT componentId,
|
|
1485
|
+
[m IN component | {pgId: m.pgId, name: m.name, type: m.type, importance: m.importance}] AS members
|
|
1486
|
+
RETURN componentId, members
|
|
1487
|
+
LIMIT 100
|
|
1488
|
+
`
|
|
1489
|
+
);
|
|
1490
|
+
return result.records.map((r, idx) => ({
|
|
1491
|
+
id: idx,
|
|
1492
|
+
members: r.get("members")
|
|
1493
|
+
}));
|
|
1494
|
+
} catch (err) {
|
|
1495
|
+
console.log("[Neo4j] getCommunities failed:", err);
|
|
1496
|
+
return [];
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
async function runCustomCypher(cypher, params) {
|
|
1500
|
+
const client = getNeo4jClient();
|
|
1501
|
+
try {
|
|
1502
|
+
const result = await client.runQuery(cypher, params);
|
|
1503
|
+
return result.records.map((r) => r.toObject());
|
|
1504
|
+
} catch (err) {
|
|
1505
|
+
console.log("[Neo4j] Custom Cypher failed:", err);
|
|
1506
|
+
throw err;
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
// src/core/intelligence/enrichment-pipeline.ts
|
|
1511
|
+
var LOG_PREFIX6 = "[OSINT:Enrich]";
|
|
1512
|
+
var _publicRecords = null;
|
|
1513
|
+
function getPublicRecords() {
|
|
1514
|
+
if (!_publicRecords) _publicRecords = new PublicRecords();
|
|
1515
|
+
return _publicRecords;
|
|
1516
|
+
}
|
|
1517
|
+
var ALL_SOURCES = ["fec", "irs990", "usaspending", "sec", "opencorporates"];
|
|
1518
|
+
async function enrichEntity(entityId, sources, depth = 1) {
|
|
1519
|
+
if (!env.OSINT_ENABLED) {
|
|
1520
|
+
console.log(`${LOG_PREFIX6} OSINT is disabled \u2014 skipping enrichment`);
|
|
1521
|
+
return {
|
|
1522
|
+
entityId,
|
|
1523
|
+
entityName: "",
|
|
1524
|
+
sourcesQueried: [],
|
|
1525
|
+
newEntitiesCreated: 0,
|
|
1526
|
+
newRelationshipsCreated: 0,
|
|
1527
|
+
errors: ["OSINT_ENABLED is false"]
|
|
1528
|
+
};
|
|
1529
|
+
}
|
|
1530
|
+
const rows = await db.select({
|
|
1531
|
+
id: graphEntities.id,
|
|
1532
|
+
name: graphEntities.name,
|
|
1533
|
+
type: graphEntities.type,
|
|
1534
|
+
attributes: graphEntities.attributes
|
|
1535
|
+
}).from(graphEntities).where(eq2(graphEntities.id, entityId)).limit(1);
|
|
1536
|
+
if (rows.length === 0) {
|
|
1537
|
+
console.log(`${LOG_PREFIX6} Entity not found: ${entityId}`);
|
|
1538
|
+
return {
|
|
1539
|
+
entityId,
|
|
1540
|
+
entityName: "",
|
|
1541
|
+
sourcesQueried: [],
|
|
1542
|
+
newEntitiesCreated: 0,
|
|
1543
|
+
newRelationshipsCreated: 0,
|
|
1544
|
+
errors: [`Entity ${entityId} not found`]
|
|
1545
|
+
};
|
|
1546
|
+
}
|
|
1547
|
+
const entity = rows[0];
|
|
1548
|
+
const entityName = entity.name;
|
|
1549
|
+
const entityType = entity.type;
|
|
1550
|
+
const attrs = entity.attributes ?? {};
|
|
1551
|
+
console.log(
|
|
1552
|
+
`${LOG_PREFIX6} Enriching "${entityName}" (${entityType}) \u2014 depth=${depth}`
|
|
1553
|
+
);
|
|
1554
|
+
const activeSources = (sources ?? [...ALL_SOURCES]).filter(
|
|
1555
|
+
(s) => ALL_SOURCES.includes(s)
|
|
1556
|
+
);
|
|
1557
|
+
const enrichedFrom = attrs.enrichedFrom ?? [];
|
|
1558
|
+
const pendingSources = activeSources.filter(
|
|
1559
|
+
(s) => !enrichedFrom.includes(s)
|
|
1560
|
+
);
|
|
1561
|
+
if (pendingSources.length === 0) {
|
|
1562
|
+
console.log(`${LOG_PREFIX6} "${entityName}" already enriched from all requested sources`);
|
|
1563
|
+
return {
|
|
1564
|
+
entityId,
|
|
1565
|
+
entityName,
|
|
1566
|
+
sourcesQueried: [],
|
|
1567
|
+
newEntitiesCreated: 0,
|
|
1568
|
+
newRelationshipsCreated: 0,
|
|
1569
|
+
errors: []
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
const enricherMap = {
|
|
1573
|
+
fec: enrichFromFEC,
|
|
1574
|
+
irs990: enrichFromIRS990,
|
|
1575
|
+
usaspending: enrichFromUSASpending,
|
|
1576
|
+
sec: enrichFromSEC,
|
|
1577
|
+
opencorporates: enrichFromOpenCorporates
|
|
1578
|
+
};
|
|
1579
|
+
let totalEntities = 0;
|
|
1580
|
+
let totalRelationships = 0;
|
|
1581
|
+
const allErrors = [];
|
|
1582
|
+
const queriedSources = [];
|
|
1583
|
+
const enrichmentPromises = pendingSources.map(async (source) => {
|
|
1584
|
+
try {
|
|
1585
|
+
const result = await enricherMap[source](entityId, entityName, entityType, attrs);
|
|
1586
|
+
queriedSources.push(source);
|
|
1587
|
+
totalEntities += result.entities;
|
|
1588
|
+
totalRelationships += result.relationships;
|
|
1589
|
+
allErrors.push(...result.errors);
|
|
1590
|
+
} catch (err) {
|
|
1591
|
+
const msg = `${source}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1592
|
+
console.error(`${LOG_PREFIX6} Top-level enricher error \u2014 ${msg}`);
|
|
1593
|
+
allErrors.push(msg);
|
|
1594
|
+
queriedSources.push(source);
|
|
1595
|
+
}
|
|
1596
|
+
});
|
|
1597
|
+
await Promise.all(enrichmentPromises);
|
|
1598
|
+
try {
|
|
1599
|
+
const updatedEnrichedFrom = [.../* @__PURE__ */ new Set([...enrichedFrom, ...queriedSources])];
|
|
1600
|
+
await db.update(graphEntities).set({
|
|
1601
|
+
attributes: {
|
|
1602
|
+
...attrs,
|
|
1603
|
+
enrichedFrom: updatedEnrichedFrom,
|
|
1604
|
+
lastEnrichedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1605
|
+
}
|
|
1606
|
+
}).where(eq2(graphEntities.id, entityId));
|
|
1607
|
+
} catch (err) {
|
|
1608
|
+
console.error(`${LOG_PREFIX6} Failed to update enrichment metadata for ${entityId}:`, err);
|
|
1609
|
+
}
|
|
1610
|
+
console.log(
|
|
1611
|
+
`${LOG_PREFIX6} Enrichment complete for "${entityName}": ${totalEntities} entities, ${totalRelationships} relationships, ${allErrors.length} errors`
|
|
1612
|
+
);
|
|
1613
|
+
return {
|
|
1614
|
+
entityId,
|
|
1615
|
+
entityName,
|
|
1616
|
+
sourcesQueried: queriedSources,
|
|
1617
|
+
newEntitiesCreated: totalEntities,
|
|
1618
|
+
newRelationshipsCreated: totalRelationships,
|
|
1619
|
+
errors: allErrors
|
|
1620
|
+
};
|
|
1621
|
+
}
|
|
1622
|
+
async function enrichFromFEC(entityId, entityName, entityType, attrs) {
|
|
1623
|
+
const pr = getPublicRecords();
|
|
1624
|
+
let entities = 0;
|
|
1625
|
+
let relationships = 0;
|
|
1626
|
+
const errors = [];
|
|
1627
|
+
if (entityType === "person") {
|
|
1628
|
+
try {
|
|
1629
|
+
const contributions = await pr.fec.getDonorLookup(entityName);
|
|
1630
|
+
console.log(
|
|
1631
|
+
`${LOG_PREFIX6} FEC: found ${contributions.length} contributions for person "${entityName}"`
|
|
1632
|
+
);
|
|
1633
|
+
const committeeMap = /* @__PURE__ */ new Map();
|
|
1634
|
+
for (const contrib of contributions) {
|
|
1635
|
+
if (!contrib.committeeId) continue;
|
|
1636
|
+
const existing = committeeMap.get(contrib.committeeId);
|
|
1637
|
+
if (existing) {
|
|
1638
|
+
existing.totalAmount += contrib.amount;
|
|
1639
|
+
existing.count += 1;
|
|
1640
|
+
} else {
|
|
1641
|
+
committeeMap.set(contrib.committeeId, {
|
|
1642
|
+
id: contrib.committeeId,
|
|
1643
|
+
name: contrib.committeeName,
|
|
1644
|
+
totalAmount: contrib.amount,
|
|
1645
|
+
count: 1
|
|
1646
|
+
});
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
for (const [fecCommitteeId, info] of committeeMap) {
|
|
1650
|
+
try {
|
|
1651
|
+
const resolved = await resolveEntity({
|
|
1652
|
+
name: info.name,
|
|
1653
|
+
type: "committee",
|
|
1654
|
+
source: "fec",
|
|
1655
|
+
identifiers: { fecId: fecCommitteeId },
|
|
1656
|
+
attributes: {
|
|
1657
|
+
fecCommitteeId
|
|
1658
|
+
}
|
|
1659
|
+
});
|
|
1660
|
+
if (resolved.isNew) entities++;
|
|
1661
|
+
await createRelationship(entityId, resolved.entityId, "donated_to", {
|
|
1662
|
+
strength: Math.min(100, Math.round(info.totalAmount / 100)),
|
|
1663
|
+
context: `${info.count} contribution(s) totaling $${info.totalAmount.toLocaleString()}`,
|
|
1664
|
+
attributes: {
|
|
1665
|
+
totalAmount: info.totalAmount,
|
|
1666
|
+
contributionCount: info.count,
|
|
1667
|
+
source: "fec"
|
|
1668
|
+
},
|
|
1669
|
+
source: "fec"
|
|
1670
|
+
});
|
|
1671
|
+
relationships++;
|
|
1672
|
+
} catch (err) {
|
|
1673
|
+
errors.push(`FEC committee resolution: ${err instanceof Error ? err.message : String(err)}`);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
} catch (err) {
|
|
1677
|
+
errors.push(`FEC donor lookup: ${err instanceof Error ? err.message : String(err)}`);
|
|
1678
|
+
}
|
|
1679
|
+
try {
|
|
1680
|
+
const candidates = await pr.fec.searchCandidates(entityName);
|
|
1681
|
+
for (const candidate of candidates.slice(0, 5)) {
|
|
1682
|
+
try {
|
|
1683
|
+
const resolved = await resolveEntity({
|
|
1684
|
+
name: candidate.name,
|
|
1685
|
+
type: "person",
|
|
1686
|
+
source: "fec",
|
|
1687
|
+
identifiers: { fecId: candidate.candidateId },
|
|
1688
|
+
attributes: {
|
|
1689
|
+
fecCandidateId: candidate.candidateId,
|
|
1690
|
+
party: candidate.party,
|
|
1691
|
+
office: candidate.office,
|
|
1692
|
+
state: candidate.state,
|
|
1693
|
+
district: candidate.district
|
|
1694
|
+
}
|
|
1695
|
+
});
|
|
1696
|
+
if (resolved.entityId !== entityId) {
|
|
1697
|
+
if (resolved.isNew) entities++;
|
|
1698
|
+
await createRelationship(entityId, resolved.entityId, "related_to", {
|
|
1699
|
+
context: `FEC candidate match: ${candidate.name} (${candidate.party})`,
|
|
1700
|
+
attributes: { source: "fec", matchType: "candidate" },
|
|
1701
|
+
source: "fec"
|
|
1702
|
+
});
|
|
1703
|
+
relationships++;
|
|
1704
|
+
}
|
|
1705
|
+
} catch (err) {
|
|
1706
|
+
errors.push(`FEC candidate resolution: ${err instanceof Error ? err.message : String(err)}`);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
} catch (err) {
|
|
1710
|
+
errors.push(`FEC candidate search: ${err instanceof Error ? err.message : String(err)}`);
|
|
1711
|
+
}
|
|
1712
|
+
} else if (entityType === "organization" || entityType === "event") {
|
|
1713
|
+
try {
|
|
1714
|
+
const committees = await pr.fec.searchCommittees(entityName);
|
|
1715
|
+
console.log(
|
|
1716
|
+
`${LOG_PREFIX6} FEC: found ${committees.length} committees for org "${entityName}"`
|
|
1717
|
+
);
|
|
1718
|
+
for (const committee of committees.slice(0, 10)) {
|
|
1719
|
+
try {
|
|
1720
|
+
const resolved = await resolveEntity({
|
|
1721
|
+
name: committee.name,
|
|
1722
|
+
type: "committee",
|
|
1723
|
+
source: "fec",
|
|
1724
|
+
identifiers: { fecId: committee.committeeId },
|
|
1725
|
+
attributes: {
|
|
1726
|
+
fecCommitteeId: committee.committeeId,
|
|
1727
|
+
designation: committee.designation,
|
|
1728
|
+
committeeType: committee.type,
|
|
1729
|
+
party: committee.party,
|
|
1730
|
+
treasurerName: committee.treasurerName
|
|
1731
|
+
}
|
|
1732
|
+
});
|
|
1733
|
+
if (resolved.isNew) entities++;
|
|
1734
|
+
await createRelationship(entityId, resolved.entityId, "related_to", {
|
|
1735
|
+
context: `FEC committee: ${committee.name} (${committee.designation})`,
|
|
1736
|
+
attributes: { source: "fec", committeeType: committee.type },
|
|
1737
|
+
source: "fec"
|
|
1738
|
+
});
|
|
1739
|
+
relationships++;
|
|
1740
|
+
try {
|
|
1741
|
+
const contributions = await pr.fec.getContributions({
|
|
1742
|
+
committeeId: committee.committeeId
|
|
1743
|
+
});
|
|
1744
|
+
const donorMap = /* @__PURE__ */ new Map();
|
|
1745
|
+
for (const c of contributions) {
|
|
1746
|
+
if (!c.contributorName) continue;
|
|
1747
|
+
const existing = donorMap.get(c.contributorName);
|
|
1748
|
+
if (existing) {
|
|
1749
|
+
existing.total += c.amount;
|
|
1750
|
+
existing.count++;
|
|
1751
|
+
} else {
|
|
1752
|
+
donorMap.set(c.contributorName, {
|
|
1753
|
+
name: c.contributorName,
|
|
1754
|
+
total: c.amount,
|
|
1755
|
+
count: 1
|
|
1756
|
+
});
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
const topDonors = [...donorMap.values()].sort((a, b) => b.total - a.total).slice(0, 10);
|
|
1760
|
+
for (const donor of topDonors) {
|
|
1761
|
+
try {
|
|
1762
|
+
const donorResolved = await resolveEntity({
|
|
1763
|
+
name: donor.name,
|
|
1764
|
+
type: "person",
|
|
1765
|
+
source: "fec",
|
|
1766
|
+
attributes: {
|
|
1767
|
+
totalContributions: donor.total,
|
|
1768
|
+
contributionCount: donor.count
|
|
1769
|
+
}
|
|
1770
|
+
});
|
|
1771
|
+
if (donorResolved.isNew) entities++;
|
|
1772
|
+
await createRelationship(
|
|
1773
|
+
donorResolved.entityId,
|
|
1774
|
+
resolved.entityId,
|
|
1775
|
+
"donated_to",
|
|
1776
|
+
{
|
|
1777
|
+
strength: Math.min(100, Math.round(donor.total / 100)),
|
|
1778
|
+
context: `${donor.count} contribution(s) totaling $${donor.total.toLocaleString()}`,
|
|
1779
|
+
attributes: {
|
|
1780
|
+
totalAmount: donor.total,
|
|
1781
|
+
contributionCount: donor.count,
|
|
1782
|
+
source: "fec"
|
|
1783
|
+
},
|
|
1784
|
+
source: "fec"
|
|
1785
|
+
}
|
|
1786
|
+
);
|
|
1787
|
+
relationships++;
|
|
1788
|
+
} catch (err) {
|
|
1789
|
+
errors.push(
|
|
1790
|
+
`FEC donor resolution (${donor.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
1791
|
+
);
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1794
|
+
} catch (err) {
|
|
1795
|
+
errors.push(
|
|
1796
|
+
`FEC contributions for ${committee.committeeId}: ${err instanceof Error ? err.message : String(err)}`
|
|
1797
|
+
);
|
|
1798
|
+
}
|
|
1799
|
+
} catch (err) {
|
|
1800
|
+
errors.push(
|
|
1801
|
+
`FEC committee resolution (${committee.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
1802
|
+
);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
} catch (err) {
|
|
1806
|
+
errors.push(`FEC committee search: ${err instanceof Error ? err.message : String(err)}`);
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
return { entities, relationships, errors };
|
|
1810
|
+
}
|
|
1811
|
+
async function enrichFromIRS990(entityId, entityName, entityType, attrs) {
|
|
1812
|
+
const pr = getPublicRecords();
|
|
1813
|
+
let entities = 0;
|
|
1814
|
+
let relationships = 0;
|
|
1815
|
+
const errors = [];
|
|
1816
|
+
if (entityType !== "organization" && entityType !== "event") {
|
|
1817
|
+
return { entities, relationships, errors };
|
|
1818
|
+
}
|
|
1819
|
+
try {
|
|
1820
|
+
const orgs = await pr.irs990.searchOrganizations(entityName);
|
|
1821
|
+
console.log(
|
|
1822
|
+
`${LOG_PREFIX6} IRS990: found ${orgs.length} nonprofits for "${entityName}"`
|
|
1823
|
+
);
|
|
1824
|
+
for (const org of orgs.slice(0, 5)) {
|
|
1825
|
+
if (!org.ein) continue;
|
|
1826
|
+
try {
|
|
1827
|
+
const detail = await pr.irs990.getOrganization(org.ein);
|
|
1828
|
+
const recentFilings = detail.filings.slice(0, 3);
|
|
1829
|
+
const financialSummary = {};
|
|
1830
|
+
if (recentFilings.length > 0) {
|
|
1831
|
+
const latest = recentFilings[0];
|
|
1832
|
+
financialSummary.latestRevenue = latest.totalRevenue;
|
|
1833
|
+
financialSummary.latestExpenses = latest.totalExpenses;
|
|
1834
|
+
financialSummary.latestAssets = latest.totalAssets;
|
|
1835
|
+
financialSummary.latestLiabilities = latest.totalLiabilities;
|
|
1836
|
+
financialSummary.latestTaxPeriod = latest.taxPeriod;
|
|
1837
|
+
financialSummary.filingCount = detail.filings.length;
|
|
1838
|
+
}
|
|
1839
|
+
const resolved = await resolveEntity({
|
|
1840
|
+
name: detail.name || org.name,
|
|
1841
|
+
type: "organization",
|
|
1842
|
+
source: "irs990",
|
|
1843
|
+
identifiers: { ein: org.ein },
|
|
1844
|
+
attributes: {
|
|
1845
|
+
ein: org.ein,
|
|
1846
|
+
city: detail.city || org.city,
|
|
1847
|
+
state: detail.state || org.state,
|
|
1848
|
+
nteeCode: detail.nteeCode || org.nteeCode,
|
|
1849
|
+
rulingDate: detail.rulingDate || org.rulingDate,
|
|
1850
|
+
...financialSummary,
|
|
1851
|
+
filings: recentFilings.map((f) => ({
|
|
1852
|
+
taxPeriod: f.taxPeriod,
|
|
1853
|
+
formType: f.formType,
|
|
1854
|
+
totalRevenue: f.totalRevenue,
|
|
1855
|
+
totalExpenses: f.totalExpenses,
|
|
1856
|
+
totalAssets: f.totalAssets,
|
|
1857
|
+
totalLiabilities: f.totalLiabilities
|
|
1858
|
+
}))
|
|
1859
|
+
}
|
|
1860
|
+
});
|
|
1861
|
+
if (resolved.isNew) entities++;
|
|
1862
|
+
if (resolved.entityId !== entityId) {
|
|
1863
|
+
await createRelationship(entityId, resolved.entityId, "related_to", {
|
|
1864
|
+
context: `IRS 990 nonprofit: ${org.name} (EIN: ${org.ein})`,
|
|
1865
|
+
attributes: {
|
|
1866
|
+
source: "irs990",
|
|
1867
|
+
ein: org.ein,
|
|
1868
|
+
...financialSummary
|
|
1869
|
+
},
|
|
1870
|
+
source: "irs990"
|
|
1871
|
+
});
|
|
1872
|
+
relationships++;
|
|
1873
|
+
} else {
|
|
1874
|
+
try {
|
|
1875
|
+
await db.update(graphEntities).set({
|
|
1876
|
+
attributes: {
|
|
1877
|
+
...attrs,
|
|
1878
|
+
ein: org.ein,
|
|
1879
|
+
nteeCode: detail.nteeCode,
|
|
1880
|
+
...financialSummary,
|
|
1881
|
+
irs990Filings: recentFilings.map((f) => ({
|
|
1882
|
+
taxPeriod: f.taxPeriod,
|
|
1883
|
+
formType: f.formType,
|
|
1884
|
+
totalRevenue: f.totalRevenue,
|
|
1885
|
+
totalExpenses: f.totalExpenses,
|
|
1886
|
+
totalAssets: f.totalAssets,
|
|
1887
|
+
totalLiabilities: f.totalLiabilities
|
|
1888
|
+
}))
|
|
1889
|
+
}
|
|
1890
|
+
}).where(eq2(graphEntities.id, entityId));
|
|
1891
|
+
} catch (err) {
|
|
1892
|
+
errors.push(
|
|
1893
|
+
`IRS990 attribute update: ${err instanceof Error ? err.message : String(err)}`
|
|
1894
|
+
);
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
} catch (err) {
|
|
1898
|
+
errors.push(
|
|
1899
|
+
`IRS990 org detail (${org.ein}): ${err instanceof Error ? err.message : String(err)}`
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
} catch (err) {
|
|
1904
|
+
errors.push(`IRS990 search: ${err instanceof Error ? err.message : String(err)}`);
|
|
1905
|
+
}
|
|
1906
|
+
return { entities, relationships, errors };
|
|
1907
|
+
}
|
|
1908
|
+
async function enrichFromUSASpending(entityId, entityName, entityType, attrs) {
|
|
1909
|
+
const pr = getPublicRecords();
|
|
1910
|
+
let entities = 0;
|
|
1911
|
+
let relationships = 0;
|
|
1912
|
+
const errors = [];
|
|
1913
|
+
try {
|
|
1914
|
+
const awards = await pr.usaspending.searchAwards({
|
|
1915
|
+
keyword: entityName
|
|
1916
|
+
});
|
|
1917
|
+
console.log(
|
|
1918
|
+
`${LOG_PREFIX6} USASpending: found ${awards.length} awards for "${entityName}"`
|
|
1919
|
+
);
|
|
1920
|
+
for (const award of awards.slice(0, 15)) {
|
|
1921
|
+
try {
|
|
1922
|
+
const awardCandidate = {
|
|
1923
|
+
name: award.description || `Award ${award.awardId}`,
|
|
1924
|
+
type: "contract",
|
|
1925
|
+
source: "usaspending",
|
|
1926
|
+
identifiers: { uei: award.recipientUei || void 0 },
|
|
1927
|
+
attributes: {
|
|
1928
|
+
awardId: award.awardId,
|
|
1929
|
+
awardType: award.type,
|
|
1930
|
+
typeDescription: award.typeDescription,
|
|
1931
|
+
totalObligationAmount: award.totalObligationAmount,
|
|
1932
|
+
totalOutlayAmount: award.totalOutlayAmount,
|
|
1933
|
+
startDate: award.startDate,
|
|
1934
|
+
endDate: award.endDate,
|
|
1935
|
+
placeOfPerformanceCity: award.placeOfPerformanceCity,
|
|
1936
|
+
placeOfPerformanceState: award.placeOfPerformanceState
|
|
1937
|
+
}
|
|
1938
|
+
};
|
|
1939
|
+
const awardResolved = await resolveEntity(awardCandidate);
|
|
1940
|
+
if (awardResolved.isNew) entities++;
|
|
1941
|
+
if (award.recipientName) {
|
|
1942
|
+
try {
|
|
1943
|
+
const recipientResolved = await resolveEntity({
|
|
1944
|
+
name: award.recipientName,
|
|
1945
|
+
type: "organization",
|
|
1946
|
+
source: "usaspending",
|
|
1947
|
+
identifiers: { uei: award.recipientUei || void 0 },
|
|
1948
|
+
attributes: {
|
|
1949
|
+
uei: award.recipientUei
|
|
1950
|
+
}
|
|
1951
|
+
});
|
|
1952
|
+
if (recipientResolved.isNew) entities++;
|
|
1953
|
+
await createRelationship(
|
|
1954
|
+
recipientResolved.entityId,
|
|
1955
|
+
awardResolved.entityId,
|
|
1956
|
+
"awarded_contract",
|
|
1957
|
+
{
|
|
1958
|
+
strength: Math.min(
|
|
1959
|
+
100,
|
|
1960
|
+
Math.round(Math.abs(award.totalObligationAmount) / 1e4)
|
|
1961
|
+
),
|
|
1962
|
+
context: `Award ${award.awardId}: $${award.totalObligationAmount.toLocaleString()}`,
|
|
1963
|
+
attributes: {
|
|
1964
|
+
amount: award.totalObligationAmount,
|
|
1965
|
+
awardType: award.type,
|
|
1966
|
+
source: "usaspending"
|
|
1967
|
+
},
|
|
1968
|
+
source: "usaspending"
|
|
1969
|
+
}
|
|
1970
|
+
);
|
|
1971
|
+
relationships++;
|
|
1972
|
+
if (recipientResolved.entityId !== entityId) {
|
|
1973
|
+
await createRelationship(entityId, recipientResolved.entityId, "related_to", {
|
|
1974
|
+
context: `Linked via USAspending award ${award.awardId}`,
|
|
1975
|
+
attributes: { source: "usaspending" },
|
|
1976
|
+
source: "usaspending"
|
|
1977
|
+
});
|
|
1978
|
+
relationships++;
|
|
1979
|
+
}
|
|
1980
|
+
} catch (err) {
|
|
1981
|
+
errors.push(
|
|
1982
|
+
`USASpending recipient (${award.recipientName}): ${err instanceof Error ? err.message : String(err)}`
|
|
1983
|
+
);
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
if (award.awardingAgencyName) {
|
|
1987
|
+
try {
|
|
1988
|
+
const agencyResolved = await resolveEntity({
|
|
1989
|
+
name: award.awardingAgencyName,
|
|
1990
|
+
type: "organization",
|
|
1991
|
+
source: "usaspending",
|
|
1992
|
+
attributes: {
|
|
1993
|
+
agencyType: "federal",
|
|
1994
|
+
subAgency: award.awardingSubAgencyName
|
|
1995
|
+
}
|
|
1996
|
+
});
|
|
1997
|
+
if (agencyResolved.isNew) entities++;
|
|
1998
|
+
await createRelationship(
|
|
1999
|
+
agencyResolved.entityId,
|
|
2000
|
+
awardResolved.entityId,
|
|
2001
|
+
"funded_by",
|
|
2002
|
+
{
|
|
2003
|
+
context: `Funded by ${award.awardingAgencyName}`,
|
|
2004
|
+
attributes: {
|
|
2005
|
+
amount: award.totalObligationAmount,
|
|
2006
|
+
source: "usaspending"
|
|
2007
|
+
},
|
|
2008
|
+
source: "usaspending"
|
|
2009
|
+
}
|
|
2010
|
+
);
|
|
2011
|
+
relationships++;
|
|
2012
|
+
} catch (err) {
|
|
2013
|
+
errors.push(
|
|
2014
|
+
`USASpending agency (${award.awardingAgencyName}): ${err instanceof Error ? err.message : String(err)}`
|
|
2015
|
+
);
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
if (award.fundingAgencyName && award.fundingAgencyName !== award.awardingAgencyName) {
|
|
2019
|
+
try {
|
|
2020
|
+
const fundingResolved = await resolveEntity({
|
|
2021
|
+
name: award.fundingAgencyName,
|
|
2022
|
+
type: "organization",
|
|
2023
|
+
source: "usaspending",
|
|
2024
|
+
attributes: { agencyType: "federal" }
|
|
2025
|
+
});
|
|
2026
|
+
if (fundingResolved.isNew) entities++;
|
|
2027
|
+
await createRelationship(
|
|
2028
|
+
fundingResolved.entityId,
|
|
2029
|
+
awardResolved.entityId,
|
|
2030
|
+
"funded_by",
|
|
2031
|
+
{
|
|
2032
|
+
context: `Funding agency: ${award.fundingAgencyName}`,
|
|
2033
|
+
attributes: { source: "usaspending" },
|
|
2034
|
+
source: "usaspending"
|
|
2035
|
+
}
|
|
2036
|
+
);
|
|
2037
|
+
relationships++;
|
|
2038
|
+
} catch (err) {
|
|
2039
|
+
errors.push(
|
|
2040
|
+
`USASpending funding agency (${award.fundingAgencyName}): ${err instanceof Error ? err.message : String(err)}`
|
|
2041
|
+
);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
} catch (err) {
|
|
2045
|
+
errors.push(
|
|
2046
|
+
`USASpending award (${award.awardId}): ${err instanceof Error ? err.message : String(err)}`
|
|
2047
|
+
);
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
} catch (err) {
|
|
2051
|
+
errors.push(`USASpending search: ${err instanceof Error ? err.message : String(err)}`);
|
|
2052
|
+
}
|
|
2053
|
+
return { entities, relationships, errors };
|
|
2054
|
+
}
|
|
2055
|
+
async function enrichFromSEC(entityId, entityName, entityType, attrs) {
|
|
2056
|
+
const pr = getPublicRecords();
|
|
2057
|
+
let entities = 0;
|
|
2058
|
+
let relationships = 0;
|
|
2059
|
+
const errors = [];
|
|
2060
|
+
if (entityType !== "organization" && entityType !== "person") {
|
|
2061
|
+
return { entities, relationships, errors };
|
|
2062
|
+
}
|
|
2063
|
+
try {
|
|
2064
|
+
const companies = await pr.sec.searchCompanies(entityName);
|
|
2065
|
+
console.log(
|
|
2066
|
+
`${LOG_PREFIX6} SEC: found ${companies.length} companies for "${entityName}"`
|
|
2067
|
+
);
|
|
2068
|
+
for (const company of companies.slice(0, 5)) {
|
|
2069
|
+
if (!company.cik) continue;
|
|
2070
|
+
try {
|
|
2071
|
+
const companyResolved = await resolveEntity({
|
|
2072
|
+
name: company.name,
|
|
2073
|
+
type: "organization",
|
|
2074
|
+
source: "sec",
|
|
2075
|
+
identifiers: { cik: company.cik },
|
|
2076
|
+
attributes: {
|
|
2077
|
+
cik: company.cik,
|
|
2078
|
+
ticker: company.ticker,
|
|
2079
|
+
exchange: company.exchange,
|
|
2080
|
+
sic: company.sic,
|
|
2081
|
+
sicDescription: company.sicDescription,
|
|
2082
|
+
stateOfIncorporation: company.stateOfIncorporation,
|
|
2083
|
+
fiscalYearEnd: company.fiscalYearEnd
|
|
2084
|
+
}
|
|
2085
|
+
});
|
|
2086
|
+
if (companyResolved.isNew) entities++;
|
|
2087
|
+
if (companyResolved.entityId !== entityId) {
|
|
2088
|
+
await createRelationship(entityId, companyResolved.entityId, "related_to", {
|
|
2089
|
+
context: `SEC EDGAR company match: ${company.name} (CIK: ${company.cik})`,
|
|
2090
|
+
attributes: { source: "sec", cik: company.cik, ticker: company.ticker },
|
|
2091
|
+
source: "sec"
|
|
2092
|
+
});
|
|
2093
|
+
relationships++;
|
|
2094
|
+
}
|
|
2095
|
+
try {
|
|
2096
|
+
const transactions = await pr.sec.getInsiderTransactions(company.cik);
|
|
2097
|
+
console.log(
|
|
2098
|
+
`${LOG_PREFIX6} SEC: found ${transactions.length} insider transactions for CIK ${company.cik}`
|
|
2099
|
+
);
|
|
2100
|
+
const ownerMap = /* @__PURE__ */ new Map();
|
|
2101
|
+
for (const tx of transactions) {
|
|
2102
|
+
if (!tx.reportingOwnerName || tx.reportingOwnerName === "See filing")
|
|
2103
|
+
continue;
|
|
2104
|
+
const key = tx.reportingOwnerCik || tx.reportingOwnerName;
|
|
2105
|
+
const existing = ownerMap.get(key);
|
|
2106
|
+
if (existing) {
|
|
2107
|
+
existing.transactionCount++;
|
|
2108
|
+
existing.totalShares += Math.abs(tx.transactionShares);
|
|
2109
|
+
existing.isDirector = existing.isDirector || tx.isDirector;
|
|
2110
|
+
existing.isOfficer = existing.isOfficer || tx.isOfficer;
|
|
2111
|
+
if (tx.officerTitle && !existing.officerTitle) {
|
|
2112
|
+
existing.officerTitle = tx.officerTitle;
|
|
2113
|
+
}
|
|
2114
|
+
} else {
|
|
2115
|
+
ownerMap.set(key, {
|
|
2116
|
+
name: tx.reportingOwnerName,
|
|
2117
|
+
cik: tx.reportingOwnerCik,
|
|
2118
|
+
isDirector: tx.isDirector,
|
|
2119
|
+
isOfficer: tx.isOfficer,
|
|
2120
|
+
officerTitle: tx.officerTitle,
|
|
2121
|
+
transactionCount: 1,
|
|
2122
|
+
totalShares: Math.abs(tx.transactionShares)
|
|
2123
|
+
});
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
const topInsiders = [...ownerMap.values()].sort((a, b) => b.totalShares - a.totalShares).slice(0, 15);
|
|
2127
|
+
for (const insider of topInsiders) {
|
|
2128
|
+
try {
|
|
2129
|
+
const insiderResolved = await resolveEntity({
|
|
2130
|
+
name: insider.name,
|
|
2131
|
+
type: "person",
|
|
2132
|
+
source: "sec",
|
|
2133
|
+
identifiers: insider.cik ? { cik: insider.cik } : void 0,
|
|
2134
|
+
attributes: {
|
|
2135
|
+
isDirector: insider.isDirector,
|
|
2136
|
+
isOfficer: insider.isOfficer,
|
|
2137
|
+
officerTitle: insider.officerTitle
|
|
2138
|
+
}
|
|
2139
|
+
});
|
|
2140
|
+
if (insiderResolved.isNew) entities++;
|
|
2141
|
+
const relType = insider.isOfficer || insider.isDirector ? "officer_of" : "insider_transaction";
|
|
2142
|
+
await createRelationship(
|
|
2143
|
+
insiderResolved.entityId,
|
|
2144
|
+
companyResolved.entityId,
|
|
2145
|
+
relType,
|
|
2146
|
+
{
|
|
2147
|
+
strength: Math.min(100, 40 + insider.transactionCount * 5),
|
|
2148
|
+
context: insider.officerTitle ? `${insider.officerTitle} \u2014 ${insider.transactionCount} transaction(s)` : `${insider.transactionCount} insider transaction(s)`,
|
|
2149
|
+
attributes: {
|
|
2150
|
+
isDirector: insider.isDirector,
|
|
2151
|
+
isOfficer: insider.isOfficer,
|
|
2152
|
+
officerTitle: insider.officerTitle,
|
|
2153
|
+
transactionCount: insider.transactionCount,
|
|
2154
|
+
totalShares: insider.totalShares,
|
|
2155
|
+
source: "sec"
|
|
2156
|
+
},
|
|
2157
|
+
source: "sec"
|
|
2158
|
+
}
|
|
2159
|
+
);
|
|
2160
|
+
relationships++;
|
|
2161
|
+
} catch (err) {
|
|
2162
|
+
errors.push(
|
|
2163
|
+
`SEC insider (${insider.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
2164
|
+
);
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
} catch (err) {
|
|
2168
|
+
errors.push(
|
|
2169
|
+
`SEC insider transactions (CIK ${company.cik}): ${err instanceof Error ? err.message : String(err)}`
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
} catch (err) {
|
|
2173
|
+
errors.push(
|
|
2174
|
+
`SEC company (${company.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
}
|
|
2178
|
+
} catch (err) {
|
|
2179
|
+
errors.push(`SEC search: ${err instanceof Error ? err.message : String(err)}`);
|
|
2180
|
+
}
|
|
2181
|
+
return { entities, relationships, errors };
|
|
2182
|
+
}
|
|
2183
|
+
async function enrichFromOpenCorporates(entityId, entityName, entityType, attrs) {
|
|
2184
|
+
const pr = getPublicRecords();
|
|
2185
|
+
let entities = 0;
|
|
2186
|
+
let relationships = 0;
|
|
2187
|
+
const errors = [];
|
|
2188
|
+
if (entityType !== "organization") {
|
|
2189
|
+
if (entityType === "person") {
|
|
2190
|
+
try {
|
|
2191
|
+
const officers = await pr.opencorporates.searchOfficers(entityName);
|
|
2192
|
+
console.log(
|
|
2193
|
+
`${LOG_PREFIX6} OpenCorporates: found ${officers.length} officer records for person "${entityName}"`
|
|
2194
|
+
);
|
|
2195
|
+
for (const officer of officers.slice(0, 10)) {
|
|
2196
|
+
if (!officer.companyName) continue;
|
|
2197
|
+
try {
|
|
2198
|
+
const companyResolved = await resolveEntity({
|
|
2199
|
+
name: officer.companyName,
|
|
2200
|
+
type: "organization",
|
|
2201
|
+
source: "opencorporates",
|
|
2202
|
+
attributes: {
|
|
2203
|
+
companyNumber: officer.companyNumber,
|
|
2204
|
+
jurisdictionCode: officer.jurisdictionCode
|
|
2205
|
+
}
|
|
2206
|
+
});
|
|
2207
|
+
if (companyResolved.isNew) entities++;
|
|
2208
|
+
await createRelationship(entityId, companyResolved.entityId, "officer_of", {
|
|
2209
|
+
strength: 60,
|
|
2210
|
+
context: `${officer.position || "Officer"} at ${officer.companyName}`,
|
|
2211
|
+
attributes: {
|
|
2212
|
+
position: officer.position,
|
|
2213
|
+
startDate: officer.startDate,
|
|
2214
|
+
endDate: officer.endDate,
|
|
2215
|
+
nationality: officer.nationality,
|
|
2216
|
+
occupation: officer.occupation,
|
|
2217
|
+
source: "opencorporates"
|
|
2218
|
+
},
|
|
2219
|
+
source: "opencorporates"
|
|
2220
|
+
});
|
|
2221
|
+
relationships++;
|
|
2222
|
+
} catch (err) {
|
|
2223
|
+
errors.push(
|
|
2224
|
+
`OpenCorporates officer company (${officer.companyName}): ${err instanceof Error ? err.message : String(err)}`
|
|
2225
|
+
);
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
} catch (err) {
|
|
2229
|
+
errors.push(
|
|
2230
|
+
`OpenCorporates officer search: ${err instanceof Error ? err.message : String(err)}`
|
|
2231
|
+
);
|
|
2232
|
+
}
|
|
2233
|
+
return { entities, relationships, errors };
|
|
2234
|
+
}
|
|
2235
|
+
return { entities, relationships, errors };
|
|
2236
|
+
}
|
|
2237
|
+
try {
|
|
2238
|
+
const companies = await pr.opencorporates.searchCompanies(entityName);
|
|
2239
|
+
console.log(
|
|
2240
|
+
`${LOG_PREFIX6} OpenCorporates: found ${companies.length} companies for "${entityName}"`
|
|
2241
|
+
);
|
|
2242
|
+
for (const company of companies.slice(0, 5)) {
|
|
2243
|
+
try {
|
|
2244
|
+
const companyResolved = await resolveEntity({
|
|
2245
|
+
name: company.name,
|
|
2246
|
+
type: "organization",
|
|
2247
|
+
source: "opencorporates",
|
|
2248
|
+
attributes: {
|
|
2249
|
+
companyNumber: company.companyNumber,
|
|
2250
|
+
jurisdictionCode: company.jurisdictionCode,
|
|
2251
|
+
incorporationDate: company.incorporationDate,
|
|
2252
|
+
dissolutionDate: company.dissolutionDate,
|
|
2253
|
+
companyType: company.companyType,
|
|
2254
|
+
status: company.status,
|
|
2255
|
+
registeredAddress: company.registeredAddress,
|
|
2256
|
+
registryUrl: company.registryUrl,
|
|
2257
|
+
openCorporatesUrl: company.openCorporatesUrl
|
|
2258
|
+
}
|
|
2259
|
+
});
|
|
2260
|
+
if (companyResolved.isNew) entities++;
|
|
2261
|
+
if (companyResolved.entityId !== entityId) {
|
|
2262
|
+
await createRelationship(entityId, companyResolved.entityId, "related_to", {
|
|
2263
|
+
context: `OpenCorporates match: ${company.name} (${company.jurisdictionCode})`,
|
|
2264
|
+
attributes: {
|
|
2265
|
+
source: "opencorporates",
|
|
2266
|
+
companyNumber: company.companyNumber,
|
|
2267
|
+
jurisdictionCode: company.jurisdictionCode
|
|
2268
|
+
},
|
|
2269
|
+
source: "opencorporates"
|
|
2270
|
+
});
|
|
2271
|
+
relationships++;
|
|
2272
|
+
}
|
|
2273
|
+
const inlineOfficers = company.officers ?? [];
|
|
2274
|
+
for (const officer of inlineOfficers.slice(0, 15)) {
|
|
2275
|
+
if (!officer.name) continue;
|
|
2276
|
+
try {
|
|
2277
|
+
const officerResolved = await resolveEntity({
|
|
2278
|
+
name: officer.name,
|
|
2279
|
+
type: "person",
|
|
2280
|
+
source: "opencorporates",
|
|
2281
|
+
attributes: {
|
|
2282
|
+
position: officer.position,
|
|
2283
|
+
nationality: officer.nationality,
|
|
2284
|
+
occupation: officer.occupation
|
|
2285
|
+
}
|
|
2286
|
+
});
|
|
2287
|
+
if (officerResolved.isNew) entities++;
|
|
2288
|
+
await createRelationship(
|
|
2289
|
+
officerResolved.entityId,
|
|
2290
|
+
companyResolved.entityId,
|
|
2291
|
+
"officer_of",
|
|
2292
|
+
{
|
|
2293
|
+
strength: 60,
|
|
2294
|
+
context: `${officer.position || "Officer"} at ${company.name}`,
|
|
2295
|
+
attributes: {
|
|
2296
|
+
position: officer.position,
|
|
2297
|
+
startDate: officer.startDate,
|
|
2298
|
+
endDate: officer.endDate,
|
|
2299
|
+
nationality: officer.nationality,
|
|
2300
|
+
occupation: officer.occupation,
|
|
2301
|
+
source: "opencorporates"
|
|
2302
|
+
},
|
|
2303
|
+
source: "opencorporates"
|
|
2304
|
+
}
|
|
2305
|
+
);
|
|
2306
|
+
relationships++;
|
|
2307
|
+
} catch (err) {
|
|
2308
|
+
errors.push(
|
|
2309
|
+
`OpenCorporates officer (${officer.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
2310
|
+
);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
if (inlineOfficers.length === 0 && company.jurisdictionCode && company.companyNumber) {
|
|
2314
|
+
try {
|
|
2315
|
+
const detailedCompany = await pr.opencorporates.getCompany(
|
|
2316
|
+
company.jurisdictionCode,
|
|
2317
|
+
company.companyNumber
|
|
2318
|
+
);
|
|
2319
|
+
for (const officer of (detailedCompany.officers ?? []).slice(0, 15)) {
|
|
2320
|
+
if (!officer.name) continue;
|
|
2321
|
+
try {
|
|
2322
|
+
const officerResolved = await resolveEntity({
|
|
2323
|
+
name: officer.name,
|
|
2324
|
+
type: "person",
|
|
2325
|
+
source: "opencorporates",
|
|
2326
|
+
attributes: {
|
|
2327
|
+
position: officer.position,
|
|
2328
|
+
nationality: officer.nationality,
|
|
2329
|
+
occupation: officer.occupation
|
|
2330
|
+
}
|
|
2331
|
+
});
|
|
2332
|
+
if (officerResolved.isNew) entities++;
|
|
2333
|
+
await createRelationship(
|
|
2334
|
+
officerResolved.entityId,
|
|
2335
|
+
companyResolved.entityId,
|
|
2336
|
+
"officer_of",
|
|
2337
|
+
{
|
|
2338
|
+
strength: 60,
|
|
2339
|
+
context: `${officer.position || "Officer"} at ${company.name}`,
|
|
2340
|
+
attributes: {
|
|
2341
|
+
position: officer.position,
|
|
2342
|
+
startDate: officer.startDate,
|
|
2343
|
+
endDate: officer.endDate,
|
|
2344
|
+
source: "opencorporates"
|
|
2345
|
+
},
|
|
2346
|
+
source: "opencorporates"
|
|
2347
|
+
}
|
|
2348
|
+
);
|
|
2349
|
+
relationships++;
|
|
2350
|
+
} catch (err) {
|
|
2351
|
+
errors.push(
|
|
2352
|
+
`OpenCorporates officer (${officer.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
2353
|
+
);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
} catch (err) {
|
|
2357
|
+
errors.push(
|
|
2358
|
+
`OpenCorporates company detail (${company.companyNumber}): ${err instanceof Error ? err.message : String(err)}`
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
} catch (err) {
|
|
2363
|
+
errors.push(
|
|
2364
|
+
`OpenCorporates company (${company.name}): ${err instanceof Error ? err.message : String(err)}`
|
|
2365
|
+
);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
} catch (err) {
|
|
2369
|
+
errors.push(`OpenCorporates search: ${err instanceof Error ? err.message : String(err)}`);
|
|
2370
|
+
}
|
|
2371
|
+
return { entities, relationships, errors };
|
|
2372
|
+
}
|
|
2373
|
+
async function batchEnrich(opts) {
|
|
2374
|
+
if (!env.OSINT_ENABLED) {
|
|
2375
|
+
console.log(`${LOG_PREFIX6} OSINT is disabled \u2014 skipping batch enrichment`);
|
|
2376
|
+
return [];
|
|
2377
|
+
}
|
|
2378
|
+
const limit = opts?.limit ?? 50;
|
|
2379
|
+
const sources = (opts?.sources ?? [...ALL_SOURCES]).filter(
|
|
2380
|
+
(s) => ALL_SOURCES.includes(s)
|
|
2381
|
+
);
|
|
2382
|
+
console.log(
|
|
2383
|
+
`${LOG_PREFIX6} Starting batch enrichment \u2014 limit=${limit}, sources=${sources.join(",")}`
|
|
2384
|
+
);
|
|
2385
|
+
const candidates = await db.select({
|
|
2386
|
+
id: graphEntities.id,
|
|
2387
|
+
name: graphEntities.name,
|
|
2388
|
+
attributes: graphEntities.attributes
|
|
2389
|
+
}).from(graphEntities).where(
|
|
2390
|
+
sql`(
|
|
2391
|
+
${graphEntities.attributes}->>'enrichedFrom' IS NULL
|
|
2392
|
+
OR NOT ${graphEntities.attributes}->'enrichedFrom' @> ${JSON.stringify(sources)}::jsonb
|
|
2393
|
+
)`
|
|
2394
|
+
).limit(limit);
|
|
2395
|
+
console.log(
|
|
2396
|
+
`${LOG_PREFIX6} Found ${candidates.length} entities needing enrichment`
|
|
2397
|
+
);
|
|
2398
|
+
const results = [];
|
|
2399
|
+
for (const candidate of candidates) {
|
|
2400
|
+
const enrichedFrom = candidate.attributes?.enrichedFrom ?? [];
|
|
2401
|
+
const missingSources = sources.filter((s) => !enrichedFrom.includes(s));
|
|
2402
|
+
if (missingSources.length === 0) continue;
|
|
2403
|
+
try {
|
|
2404
|
+
const result = await enrichEntity(candidate.id, missingSources, 1);
|
|
2405
|
+
results.push(result);
|
|
2406
|
+
} catch (err) {
|
|
2407
|
+
console.error(
|
|
2408
|
+
`${LOG_PREFIX6} Batch enrichment failed for "${candidate.name}":`,
|
|
2409
|
+
err
|
|
2410
|
+
);
|
|
2411
|
+
results.push({
|
|
2412
|
+
entityId: candidate.id,
|
|
2413
|
+
entityName: candidate.name,
|
|
2414
|
+
sourcesQueried: missingSources,
|
|
2415
|
+
newEntitiesCreated: 0,
|
|
2416
|
+
newRelationshipsCreated: 0,
|
|
2417
|
+
errors: [err instanceof Error ? err.message : String(err)]
|
|
2418
|
+
});
|
|
2419
|
+
}
|
|
2420
|
+
}
|
|
2421
|
+
const totalEntities = results.reduce((s, r) => s + r.newEntitiesCreated, 0);
|
|
2422
|
+
const totalRels = results.reduce((s, r) => s + r.newRelationshipsCreated, 0);
|
|
2423
|
+
const totalErrors = results.reduce((s, r) => s + r.errors.length, 0);
|
|
2424
|
+
console.log(
|
|
2425
|
+
`${LOG_PREFIX6} Batch enrichment complete: ${results.length} entities processed, ${totalEntities} new entities, ${totalRels} new relationships, ${totalErrors} errors`
|
|
2426
|
+
);
|
|
2427
|
+
return results;
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
export {
|
|
2431
|
+
PublicRecords,
|
|
2432
|
+
createPublicRecords,
|
|
2433
|
+
findEntitiesByName,
|
|
2434
|
+
getNeighbors,
|
|
2435
|
+
findShortestPath,
|
|
2436
|
+
getCommunities,
|
|
2437
|
+
runCustomCypher,
|
|
2438
|
+
enrichEntity,
|
|
2439
|
+
batchEnrich
|
|
2440
|
+
};
|
|
2441
|
+
//# sourceMappingURL=chunk-SVAPX2XN.js.map
|