@tiangong-ai/cli 0.0.51 → 0.0.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/AGENTS.md +22 -2
  2. package/README.md +171 -7
  3. package/dist/cli.js +10 -1
  4. package/dist/cli.js.map +1 -1
  5. package/dist/data/builtins.d.ts +1 -0
  6. package/dist/data/builtins.js +40 -0
  7. package/dist/data/builtins.js.map +1 -0
  8. package/dist/data/catalog.d.ts +25 -0
  9. package/dist/data/catalog.js +305 -0
  10. package/dist/data/catalog.js.map +1 -0
  11. package/dist/data/commands.d.ts +9 -0
  12. package/dist/data/commands.js +307 -0
  13. package/dist/data/commands.js.map +1 -0
  14. package/dist/data/connectors/airnow-hourly-observations.d.ts +2 -0
  15. package/dist/data/connectors/airnow-hourly-observations.js +522 -0
  16. package/dist/data/connectors/airnow-hourly-observations.js.map +1 -0
  17. package/dist/data/connectors/airnow-hourly-observations.schemas.d.ts +299 -0
  18. package/dist/data/connectors/airnow-hourly-observations.schemas.js +188 -0
  19. package/dist/data/connectors/airnow-hourly-observations.schemas.js.map +1 -0
  20. package/dist/data/connectors/bluesky-public-posts.d.ts +2 -0
  21. package/dist/data/connectors/bluesky-public-posts.js +705 -0
  22. package/dist/data/connectors/bluesky-public-posts.js.map +1 -0
  23. package/dist/data/connectors/bluesky-public-posts.schemas.d.ts +733 -0
  24. package/dist/data/connectors/bluesky-public-posts.schemas.js +422 -0
  25. package/dist/data/connectors/bluesky-public-posts.schemas.js.map +1 -0
  26. package/dist/data/connectors/epa-eis-records.d.ts +2 -0
  27. package/dist/data/connectors/epa-eis-records.js +566 -0
  28. package/dist/data/connectors/epa-eis-records.js.map +1 -0
  29. package/dist/data/connectors/epa-eis-records.schemas.d.ts +199 -0
  30. package/dist/data/connectors/epa-eis-records.schemas.js +146 -0
  31. package/dist/data/connectors/epa-eis-records.schemas.js.map +1 -0
  32. package/dist/data/connectors/federal-register-documents.d.ts +2 -0
  33. package/dist/data/connectors/federal-register-documents.js +517 -0
  34. package/dist/data/connectors/federal-register-documents.js.map +1 -0
  35. package/dist/data/connectors/federal-register-documents.schemas.d.ts +357 -0
  36. package/dist/data/connectors/federal-register-documents.schemas.js +243 -0
  37. package/dist/data/connectors/federal-register-documents.schemas.js.map +1 -0
  38. package/dist/data/connectors/gdelt-doc-search.d.ts +2 -0
  39. package/dist/data/connectors/gdelt-doc-search.js +658 -0
  40. package/dist/data/connectors/gdelt-doc-search.js.map +1 -0
  41. package/dist/data/connectors/gdelt-doc-search.schemas.d.ts +446 -0
  42. package/dist/data/connectors/gdelt-doc-search.schemas.js +330 -0
  43. package/dist/data/connectors/gdelt-doc-search.schemas.js.map +1 -0
  44. package/dist/data/connectors/gdelt-file-feeds.d.ts +4 -0
  45. package/dist/data/connectors/gdelt-file-feeds.js +591 -0
  46. package/dist/data/connectors/gdelt-file-feeds.js.map +1 -0
  47. package/dist/data/connectors/gdelt-file-feeds.schemas.d.ts +40 -0
  48. package/dist/data/connectors/gdelt-file-feeds.schemas.js +256 -0
  49. package/dist/data/connectors/gdelt-file-feeds.schemas.js.map +1 -0
  50. package/dist/data/connectors/nasa-firms-fire.d.ts +2 -0
  51. package/dist/data/connectors/nasa-firms-fire.js +578 -0
  52. package/dist/data/connectors/nasa-firms-fire.js.map +1 -0
  53. package/dist/data/connectors/nasa-firms-fire.schemas.d.ts +369 -0
  54. package/dist/data/connectors/nasa-firms-fire.schemas.js +238 -0
  55. package/dist/data/connectors/nasa-firms-fire.schemas.js.map +1 -0
  56. package/dist/data/connectors/open-meteo-air-quality.d.ts +2 -0
  57. package/dist/data/connectors/open-meteo-air-quality.js +401 -0
  58. package/dist/data/connectors/open-meteo-air-quality.js.map +1 -0
  59. package/dist/data/connectors/open-meteo-air-quality.schemas.d.ts +335 -0
  60. package/dist/data/connectors/open-meteo-air-quality.schemas.js +225 -0
  61. package/dist/data/connectors/open-meteo-air-quality.schemas.js.map +1 -0
  62. package/dist/data/connectors/open-meteo-flood.d.ts +2 -0
  63. package/dist/data/connectors/open-meteo-flood.js +446 -0
  64. package/dist/data/connectors/open-meteo-flood.js.map +1 -0
  65. package/dist/data/connectors/open-meteo-flood.schemas.d.ts +388 -0
  66. package/dist/data/connectors/open-meteo-flood.schemas.js +227 -0
  67. package/dist/data/connectors/open-meteo-flood.schemas.js.map +1 -0
  68. package/dist/data/connectors/open-meteo-historical-weather.d.ts +2 -0
  69. package/dist/data/connectors/open-meteo-historical-weather.js +468 -0
  70. package/dist/data/connectors/open-meteo-historical-weather.js.map +1 -0
  71. package/dist/data/connectors/open-meteo-historical-weather.schemas.d.ts +420 -0
  72. package/dist/data/connectors/open-meteo-historical-weather.schemas.js +282 -0
  73. package/dist/data/connectors/open-meteo-historical-weather.schemas.js.map +1 -0
  74. package/dist/data/connectors/openaq-air-quality.d.ts +2 -0
  75. package/dist/data/connectors/openaq-air-quality.js +734 -0
  76. package/dist/data/connectors/openaq-air-quality.js.map +1 -0
  77. package/dist/data/connectors/openaq-air-quality.schemas.d.ts +1067 -0
  78. package/dist/data/connectors/openaq-air-quality.schemas.js +512 -0
  79. package/dist/data/connectors/openaq-air-quality.schemas.js.map +1 -0
  80. package/dist/data/connectors/regulations-gov-attachments.d.ts +2 -0
  81. package/dist/data/connectors/regulations-gov-attachments.js +486 -0
  82. package/dist/data/connectors/regulations-gov-attachments.js.map +1 -0
  83. package/dist/data/connectors/regulations-gov-attachments.schemas.d.ts +270 -0
  84. package/dist/data/connectors/regulations-gov-attachments.schemas.js +181 -0
  85. package/dist/data/connectors/regulations-gov-attachments.schemas.js.map +1 -0
  86. package/dist/data/connectors/regulations-gov-comments.d.ts +2 -0
  87. package/dist/data/connectors/regulations-gov-comments.js +718 -0
  88. package/dist/data/connectors/regulations-gov-comments.js.map +1 -0
  89. package/dist/data/connectors/regulations-gov-comments.schemas.d.ts +717 -0
  90. package/dist/data/connectors/regulations-gov-comments.schemas.js +389 -0
  91. package/dist/data/connectors/regulations-gov-comments.schemas.js.map +1 -0
  92. package/dist/data/connectors/usbr-project-records.d.ts +2 -0
  93. package/dist/data/connectors/usbr-project-records.js +467 -0
  94. package/dist/data/connectors/usbr-project-records.js.map +1 -0
  95. package/dist/data/connectors/usbr-project-records.schemas.d.ts +194 -0
  96. package/dist/data/connectors/usbr-project-records.schemas.js +137 -0
  97. package/dist/data/connectors/usbr-project-records.schemas.js.map +1 -0
  98. package/dist/data/connectors/usbr-rise.d.ts +2 -0
  99. package/dist/data/connectors/usbr-rise.js +625 -0
  100. package/dist/data/connectors/usbr-rise.js.map +1 -0
  101. package/dist/data/connectors/usbr-rise.schemas.d.ts +623 -0
  102. package/dist/data/connectors/usbr-rise.schemas.js +407 -0
  103. package/dist/data/connectors/usbr-rise.schemas.js.map +1 -0
  104. package/dist/data/connectors/usgs-water-instantaneous-values.d.ts +2 -0
  105. package/dist/data/connectors/usgs-water-instantaneous-values.js +521 -0
  106. package/dist/data/connectors/usgs-water-instantaneous-values.js.map +1 -0
  107. package/dist/data/connectors/usgs-water-instantaneous-values.schemas.d.ts +496 -0
  108. package/dist/data/connectors/usgs-water-instantaneous-values.schemas.js +309 -0
  109. package/dist/data/connectors/usgs-water-instantaneous-values.schemas.js.map +1 -0
  110. package/dist/data/connectors/youtube-public-content.d.ts +2 -0
  111. package/dist/data/connectors/youtube-public-content.js +974 -0
  112. package/dist/data/connectors/youtube-public-content.js.map +1 -0
  113. package/dist/data/connectors/youtube-public-content.schemas.d.ts +773 -0
  114. package/dist/data/connectors/youtube-public-content.schemas.js +567 -0
  115. package/dist/data/connectors/youtube-public-content.schemas.js.map +1 -0
  116. package/dist/data/contracts.d.ts +374 -0
  117. package/dist/data/contracts.js +10 -0
  118. package/dist/data/contracts.js.map +1 -0
  119. package/dist/data/runtime/artifacts.d.ts +6 -0
  120. package/dist/data/runtime/artifacts.js +103 -0
  121. package/dist/data/runtime/artifacts.js.map +1 -0
  122. package/dist/data/runtime/bounded-http.d.ts +10 -0
  123. package/dist/data/runtime/bounded-http.js +403 -0
  124. package/dist/data/runtime/bounded-http.js.map +1 -0
  125. package/dist/data/runtime/canonical-json.d.ts +3 -0
  126. package/dist/data/runtime/canonical-json.js +59 -0
  127. package/dist/data/runtime/canonical-json.js.map +1 -0
  128. package/dist/data/runtime/credentials.d.ts +14 -0
  129. package/dist/data/runtime/credentials.js +67 -0
  130. package/dist/data/runtime/credentials.js.map +1 -0
  131. package/dist/data/runtime/csv.d.ts +4 -0
  132. package/dist/data/runtime/csv.js +51 -0
  133. package/dist/data/runtime/csv.js.map +1 -0
  134. package/dist/data/runtime/errors.d.ts +22 -0
  135. package/dist/data/runtime/errors.js +101 -0
  136. package/dist/data/runtime/errors.js.map +1 -0
  137. package/dist/data/runtime/execute.d.ts +11 -0
  138. package/dist/data/runtime/execute.js +358 -0
  139. package/dist/data/runtime/execute.js.map +1 -0
  140. package/dist/data/runtime/receipts.d.ts +13 -0
  141. package/dist/data/runtime/receipts.js +38 -0
  142. package/dist/data/runtime/receipts.js.map +1 -0
  143. package/dist/data/schemas/catalog.v1.json +73 -0
  144. package/dist/data/schemas/core-receipt.v1.json +117 -0
  145. package/dist/data/schemas/describe.v1.json +24 -0
  146. package/dist/data/schemas/discovery.v1.json +130 -0
  147. package/dist/data/schemas/doctor.v1.json +40 -0
  148. package/dist/data/schemas/error.v1.json +48 -0
  149. package/dist/data/schemas/manifest.v1.json +174 -0
  150. package/dist/data/schemas/run-request.v1.json +37 -0
  151. package/dist/data/schemas/run-result.v1.json +77 -0
  152. package/dist/data/schemas.d.ts +27 -0
  153. package/dist/data/schemas.js +69 -0
  154. package/dist/data/schemas.js.map +1 -0
  155. package/dist/env.js.map +1 -1
  156. package/dist/errors.js.map +1 -1
  157. package/dist/http.js.map +1 -1
  158. package/dist/kb/config.js.map +1 -1
  159. package/dist/kb/pipeline-health.js.map +1 -1
  160. package/dist/research/orchestration.js +54 -7
  161. package/dist/research/orchestration.js.map +1 -1
  162. package/dist/research/workspace/acquisition.js.map +1 -1
  163. package/dist/research/workspace/audit-bundle.js +7 -2
  164. package/dist/research/workspace/audit-bundle.js.map +1 -1
  165. package/dist/research/workspace/broker.js.map +1 -1
  166. package/dist/research/workspace/constants.js.map +1 -1
  167. package/dist/research/workspace/credentials.d.ts +3 -0
  168. package/dist/research/workspace/credentials.js +20 -3
  169. package/dist/research/workspace/credentials.js.map +1 -1
  170. package/dist/research/workspace/data-evidence-adapter.d.ts +60 -0
  171. package/dist/research/workspace/data-evidence-adapter.js +314 -0
  172. package/dist/research/workspace/data-evidence-adapter.js.map +1 -0
  173. package/dist/research/workspace/discovery-status.js +10 -7
  174. package/dist/research/workspace/discovery-status.js.map +1 -1
  175. package/dist/research/workspace/discovery.js +12 -11
  176. package/dist/research/workspace/discovery.js.map +1 -1
  177. package/dist/research/workspace/evidence-ledger.d.ts +8 -1
  178. package/dist/research/workspace/evidence-ledger.js +46 -2
  179. package/dist/research/workspace/evidence-ledger.js.map +1 -1
  180. package/dist/research/workspace/evidence.d.ts +31 -0
  181. package/dist/research/workspace/evidence.js +77 -0
  182. package/dist/research/workspace/evidence.js.map +1 -1
  183. package/dist/research/workspace/executor.js.map +1 -1
  184. package/dist/research/workspace/external-skills.d.ts +49 -49
  185. package/dist/research/workspace/external-skills.js.map +1 -1
  186. package/dist/research/workspace/platform-capabilities.js.map +1 -1
  187. package/dist/research/workspace/preflight.d.ts +4 -4
  188. package/dist/research/workspace/preflight.js +33 -8
  189. package/dist/research/workspace/preflight.js.map +1 -1
  190. package/dist/research/workspace/projects.js.map +1 -1
  191. package/dist/research/workspace/publication.js.map +1 -1
  192. package/dist/research/workspace/research-policy-wizard.js.map +1 -1
  193. package/dist/research/workspace/research-policy.js.map +1 -1
  194. package/dist/research/workspace/runtime.d.ts +7 -0
  195. package/dist/research/workspace/runtime.js +77 -20
  196. package/dist/research/workspace/runtime.js.map +1 -1
  197. package/dist/research/workspace/sanitization.js.map +1 -1
  198. package/dist/research/workspace/schemas.js +1 -1
  199. package/dist/research/workspace/schemas.js.map +1 -1
  200. package/dist/research/workspace/scientific-design.js.map +1 -1
  201. package/dist/research/workspace/scientific-review.js.map +1 -1
  202. package/dist/research/workspace/setup-catalog.d.ts +9 -9
  203. package/dist/research/workspace/setup-catalog.js +2 -2
  204. package/dist/research/workspace/setup-catalog.js.map +1 -1
  205. package/dist/research/workspace/setup-declarative.js.map +1 -1
  206. package/dist/research/workspace/setup-instructions.d.ts +42 -0
  207. package/dist/research/workspace/setup-instructions.js +413 -0
  208. package/dist/research/workspace/setup-instructions.js.map +1 -0
  209. package/dist/research/workspace/setup-invocation.js.map +1 -1
  210. package/dist/research/workspace/setup-wizard.js +7 -0
  211. package/dist/research/workspace/setup-wizard.js.map +1 -1
  212. package/dist/research/workspace/setup.d.ts +18 -15
  213. package/dist/research/workspace/setup.js +68 -3
  214. package/dist/research/workspace/setup.js.map +1 -1
  215. package/dist/research/workspace/workspace.js.map +1 -1
  216. package/package.json +3 -3
@@ -0,0 +1,658 @@
1
+ import { DATA_MANIFEST_SCHEMA_VERSION } from "../contracts.js";
2
+ import { DataRuntimeError } from "../runtime/errors.js";
3
+ import { GDELT_DOC_SEARCH_INPUT_SCHEMA, GDELT_DOC_SEARCH_OUTPUT_SCHEMA, } from "./gdelt-doc-search.schemas.js";
4
+ const DOC_PATH = "/api/v2/doc/doc";
5
+ const TIMELINE_MODES = new Set([
6
+ "timelinevol",
7
+ "timelinevolraw",
8
+ "timelinetone",
9
+ "timelinelang",
10
+ "timelinesourcecountry",
11
+ ]);
12
+ const RELATIVE_MAXIMUMS = {
13
+ minutes: 527_040,
14
+ hours: 8_784,
15
+ days: 366,
16
+ weeks: 52,
17
+ months: 12,
18
+ years: 1,
19
+ };
20
+ const RELATIVE_SUFFIXES = {
21
+ minutes: "min",
22
+ hours: "h",
23
+ days: "d",
24
+ weeks: "w",
25
+ months: "m",
26
+ years: "y",
27
+ };
28
+ export const gdeltDocSearchConnector = {
29
+ schemaVersion: DATA_MANIFEST_SCHEMA_VERSION,
30
+ capabilityId: "gdelt.doc-search",
31
+ capabilityVersion: "1.0.0",
32
+ minimumCliVersion: "0.0.55",
33
+ provider: { providerId: "gdelt", name: "GDELT Project" },
34
+ sourceCategory: "global-news-metadata",
35
+ endpoints: [
36
+ {
37
+ endpointId: "gdelt-doc-api",
38
+ baseUrl: "https://api.gdeltproject.org",
39
+ pathPrefixes: ["/api/v2/doc/"],
40
+ allowedMethods: ["GET"],
41
+ allowedContentTypes: ["application/json"],
42
+ },
43
+ ],
44
+ license: {
45
+ name: "GDELT Project data",
46
+ url: "https://www.gdeltproject.org/about.html",
47
+ restrictions: [
48
+ "GDELT derives signals automatically from monitored news and other open sources.",
49
+ "Users remain responsible for source-specific rights when following or reusing linked content.",
50
+ "This connector returns DOC metadata and aggregate timelines, not article bodies or media bytes.",
51
+ ],
52
+ },
53
+ credentials: [],
54
+ limits: {
55
+ timeoutMs: 45_000,
56
+ maxRequestBytes: 4_096,
57
+ maxResponseBytes: 20_000_000,
58
+ maxPages: 20,
59
+ maxRecords: 5_000,
60
+ maxRetries: 4,
61
+ maxRetryDelayMs: 120_000,
62
+ maxRedirects: 2,
63
+ },
64
+ diagnostics: { static: true, live: false },
65
+ freshness: {
66
+ kind: "provider-current",
67
+ description: "DOC results reflect the provider's rolling index at request time.",
68
+ },
69
+ limitations: [
70
+ "Non-timeline DOC modes only consider the final three months of a longer selected window under provider behavior.",
71
+ "Language translation, entity extraction, tone, and topical coding are automated and can be wrong.",
72
+ "Monitored-source coverage and reporting volume vary across countries, languages, and time.",
73
+ ],
74
+ discovery: {
75
+ source: {
76
+ maintainedBy: "The GDELT Project",
77
+ summary: "GDELT DOC 2.0 global news search metadata and aggregate timelines.",
78
+ description: "The GDELT DOC 2.0 API searches a rolling multilingual news index and exposes article-link metadata or aggregate timelines derived through automated processing.",
79
+ coverage: {
80
+ geographic: "Global monitored news sources, with uneven source and language coverage.",
81
+ temporal: "Provider-searchable DOC holdings; non-timeline modes only consider the final three months of a longer selected window.",
82
+ granularity: "One indexed article metadata item or one timestamped aggregate timeline point.",
83
+ },
84
+ },
85
+ summary: "Search recent GDELT news metadata or bounded aggregate news timelines.",
86
+ description: "Use a closed DOC 2.0 JSON surface with an optional rolling or absolute window; repeated domain filters are split into bounded requests and output is normalized under the common runtime.",
87
+ provides: [
88
+ "Article-link metadata for bounded recent-news queries.",
89
+ "Volume, raw-volume, tone, language, or source-country timeline series.",
90
+ "Tone-distribution histogram bins for tonechart mode.",
91
+ "Local rejection of unsupported site:/inurl: syntax and unparenthesized boolean OR groups.",
92
+ "Explicit query parameters and machine-verifiable source observations.",
93
+ ],
94
+ doesNotProvide: [
95
+ "Article full text, article body downloads, image bytes, or archived source files.",
96
+ "Representative samples of all news, population opinion, or ground-truth facts.",
97
+ "Causal conclusions or validation that an automatically extracted claim or event occurred.",
98
+ ],
99
+ selectionHints: [
100
+ "Choose DOC search for recent article discovery or aggregate attention/tone trends.",
101
+ "Choose the Events, Mentions, or GKG file capability for structured 15-minute feed records.",
102
+ "Treat returned links as candidates for separately governed source retrieval and verification.",
103
+ "Use exactDomains for domainis: filters and domains for suffix-matching domain: filters; each value becomes a separate bounded query batch.",
104
+ ],
105
+ typicalUseCases: [
106
+ "Find recent article metadata matching a topic and country filter.",
107
+ "Compare the temporal shape of monitored-news attention within a bounded window.",
108
+ ],
109
+ sourceDocumentation: [
110
+ {
111
+ title: "GDELT DOC 2.0 API",
112
+ url: "https://blog.gdeltproject.org/gdelt-doc-2-0-api-debuts/",
113
+ },
114
+ { title: "About GDELT", url: "https://www.gdeltproject.org/about.html" },
115
+ ],
116
+ },
117
+ operations: [
118
+ {
119
+ operationId: "search",
120
+ operationVersion: "1.0.0",
121
+ summary: "Search recent article metadata or one supported GDELT DOC timeline.",
122
+ description: "Validates one bounded search window and mode-specific parameters, calls the JSON endpoint, and normalizes article metadata or timeline points without retrieving linked content.",
123
+ inputSchema: GDELT_DOC_SEARCH_INPUT_SCHEMA,
124
+ outputSchema: GDELT_DOC_SEARCH_OUTPUT_SCHEMA,
125
+ execute: executeGdeltDocSearch,
126
+ },
127
+ ],
128
+ };
129
+ async function executeGdeltDocSearch(context) {
130
+ const query = normalizeQuery(context.input);
131
+ const effectiveQueries = query.domainFilters.length === 0
132
+ ? [query.query]
133
+ : query.domainFilters.map((filter) => composeDomainQuery(query.query, filter));
134
+ if (effectiveQueries.length > context.limits.maxPages) {
135
+ throw new DataRuntimeError("invalid-request", "GDELT DOC split-domain queries exceed the effective request limit.", { details: { queryCount: effectiveQueries.length, maxQueries: context.limits.maxPages } });
136
+ }
137
+ const batches = [];
138
+ const failures = [];
139
+ for (const effectiveQuery of effectiveQueries) {
140
+ try {
141
+ const response = await context.http.request({
142
+ endpointId: "gdelt-doc-api",
143
+ method: "GET",
144
+ path: DOC_PATH,
145
+ query: buildProviderQuery(query, effectiveQuery),
146
+ });
147
+ const payload = requireObject(response.json(), "GDELT DOC response");
148
+ validateModePayload(payload, query.mode);
149
+ batches.push({ query: effectiveQuery, payload, observation: response.observation });
150
+ }
151
+ catch (error) {
152
+ if (!query.continueOnQueryError)
153
+ throw normalizeProviderFailure(error);
154
+ const normalized = normalizeProviderFailure(error);
155
+ failures.push({ query: effectiveQuery, code: normalized.code, error: normalized });
156
+ }
157
+ }
158
+ if (batches.length === 0) {
159
+ throw new DataRuntimeError("provider-response-invalid", "All GDELT DOC query batches failed.", {
160
+ details: { failedQueries: failures.map((item) => item.query) },
161
+ });
162
+ }
163
+ const partial = failures.length > 0;
164
+ const queryDetails = normalizeQueryDetails(batches[0].payload.query_details);
165
+ const batchQueries = batches.map((batch) => ({
166
+ query: batch.query,
167
+ queryDetails: normalizeQueryDetails(batch.payload.query_details),
168
+ }));
169
+ const queryErrors = failures.map(({ query: failedQuery, code }) => ({
170
+ query: failedQuery,
171
+ code,
172
+ }));
173
+ const observations = batches.map((batch, index) => ({
174
+ ...batch.observation,
175
+ sourceId: `doc:batch:${index + 1}`,
176
+ }));
177
+ const errors = partial
178
+ ? [
179
+ {
180
+ code: "partial-result",
181
+ message: "One or more split GDELT DOC query batches failed.",
182
+ retryable: failures.some(({ error }) => error instanceof DataRuntimeError && (error.options.retryable ?? false)),
183
+ userActionRequired: false,
184
+ details: { failedQueries: failures.map((item) => item.query) },
185
+ },
186
+ ]
187
+ : [];
188
+ if (query.mode === "artlist") {
189
+ const cap = Math.min(context.limits.maxRecords, query.maxRecords ?? 75);
190
+ const articles = [];
191
+ const seen = new Set();
192
+ let availableArticleCount = 0;
193
+ for (const batch of batches) {
194
+ for (const value of requireArray(batch.payload.articles, "articles")) {
195
+ const article = normalizeArticle(value, availableArticleCount, batch.query);
196
+ if (!article)
197
+ continue;
198
+ const identity = articleIdentity(article);
199
+ if (seen.has(identity))
200
+ continue;
201
+ seen.add(identity);
202
+ availableArticleCount += 1;
203
+ if (articles.length < cap)
204
+ articles.push(article);
205
+ }
206
+ }
207
+ const truncated = availableArticleCount > articles.length;
208
+ return {
209
+ status: partial ? "partial" : "success",
210
+ data: {
211
+ source: { providerId: "gdelt", endpoint: DOC_PATH, metadataOnly: true },
212
+ query,
213
+ kind: "articles",
214
+ queryDetails,
215
+ batchQueries,
216
+ queryErrors,
217
+ articles,
218
+ timelines: [],
219
+ toneBins: [],
220
+ stopReason: partial
221
+ ? "partial"
222
+ : articles.length === 0
223
+ ? "no-results"
224
+ : truncated
225
+ ? "max-records"
226
+ : "completed",
227
+ },
228
+ summary: {
229
+ recordCount: articles.length,
230
+ pageCount: batches.length,
231
+ chunkCount: 0,
232
+ truncated,
233
+ completeness: partial ? "partial" : "complete",
234
+ ...(partial
235
+ ? {
236
+ missing: [
237
+ { kind: "range", identifiers: failures.map((item) => item.query) },
238
+ ],
239
+ }
240
+ : {}),
241
+ },
242
+ warnings: discoveryWarnings(truncated),
243
+ errors,
244
+ observations,
245
+ };
246
+ }
247
+ if (query.mode === "tonechart") {
248
+ const toneBins = batches.flatMap((batch) => normalizeToneBins(batch.payload, batch.query));
249
+ const capped = toneBins.slice(0, context.limits.maxRecords);
250
+ const truncated = toneBins.length > capped.length;
251
+ return {
252
+ status: partial ? "partial" : "success",
253
+ data: {
254
+ source: { providerId: "gdelt", endpoint: DOC_PATH, metadataOnly: true },
255
+ query,
256
+ kind: "tone-chart",
257
+ queryDetails,
258
+ batchQueries,
259
+ queryErrors,
260
+ articles: [],
261
+ timelines: [],
262
+ toneBins: capped,
263
+ stopReason: partial
264
+ ? "partial"
265
+ : capped.length === 0
266
+ ? "no-results"
267
+ : truncated
268
+ ? "max-records"
269
+ : "completed",
270
+ },
271
+ summary: {
272
+ recordCount: capped.length,
273
+ pageCount: batches.length,
274
+ chunkCount: batches.length,
275
+ truncated,
276
+ completeness: partial ? "partial" : "complete",
277
+ ...(partial
278
+ ? {
279
+ missing: [
280
+ { kind: "range", identifiers: failures.map((item) => item.query) },
281
+ ],
282
+ }
283
+ : {}),
284
+ },
285
+ warnings: discoveryWarnings(truncated),
286
+ errors,
287
+ observations,
288
+ };
289
+ }
290
+ let remaining = context.limits.maxRecords;
291
+ let rawPointCount = 0;
292
+ const timelines = batches.flatMap((batch) => requireArray(batch.payload.timeline, "timeline").map((value, seriesIndex) => {
293
+ const series = requireObject(value, `timeline[${seriesIndex}]`);
294
+ const rawData = requireArray(series.data, `timeline[${seriesIndex}].data`);
295
+ rawPointCount += rawData.length;
296
+ const data = rawData
297
+ .slice(0, remaining)
298
+ .map((point, pointIndex) => normalizeTimelinePoint(point, `timeline[${seriesIndex}].data[${pointIndex}]`));
299
+ remaining -= data.length;
300
+ return {
301
+ query: batch.query,
302
+ series: requireNonBlankString(series.series, `timeline[${seriesIndex}].series`),
303
+ data,
304
+ };
305
+ }));
306
+ const recordCount = context.limits.maxRecords - remaining;
307
+ const truncated = rawPointCount > recordCount;
308
+ return {
309
+ status: partial ? "partial" : "success",
310
+ data: {
311
+ source: { providerId: "gdelt", endpoint: DOC_PATH, metadataOnly: true },
312
+ query,
313
+ kind: "timeline",
314
+ queryDetails,
315
+ batchQueries,
316
+ queryErrors,
317
+ articles: [],
318
+ timelines,
319
+ toneBins: [],
320
+ stopReason: partial
321
+ ? "partial"
322
+ : recordCount === 0
323
+ ? "no-results"
324
+ : truncated
325
+ ? "max-records"
326
+ : "completed",
327
+ },
328
+ summary: {
329
+ recordCount,
330
+ pageCount: batches.length,
331
+ chunkCount: timelines.length,
332
+ truncated,
333
+ completeness: partial ? "partial" : "complete",
334
+ ...(partial
335
+ ? {
336
+ missing: [{ kind: "range", identifiers: failures.map((item) => item.query) }],
337
+ }
338
+ : {}),
339
+ },
340
+ warnings: discoveryWarnings(truncated),
341
+ errors,
342
+ observations,
343
+ };
344
+ }
345
+ function normalizeQuery(input) {
346
+ const query = input.query.trim().replace(/\s+/g, " ");
347
+ if (!query || /[\u0000-\u001f\u007f]/.test(input.query)) {
348
+ throw new DataRuntimeError("invalid-request", "GDELT DOC query must be non-blank text without control characters.");
349
+ }
350
+ const hasRelative = input.relativeWindow !== undefined;
351
+ const hasAbsolute = input.absoluteWindow !== undefined;
352
+ if (hasRelative && hasAbsolute) {
353
+ throw new DataRuntimeError("invalid-request", "GDELT DOC search accepts relativeWindow or absoluteWindow, not both.");
354
+ }
355
+ let relativeWindow = null;
356
+ let absoluteWindow = null;
357
+ if (input.relativeWindow) {
358
+ if (input.relativeWindow.unit === "minutes" && input.relativeWindow.value < 15) {
359
+ throw new DataRuntimeError("invalid-request", "GDELT DOC relativeWindow must span at least 15 minutes.");
360
+ }
361
+ relativeWindow = { ...input.relativeWindow };
362
+ }
363
+ if (input.absoluteWindow) {
364
+ const from = parseRfc3339(input.absoluteWindow.from, "absoluteWindow.from");
365
+ const to = parseRfc3339(input.absoluteWindow.to, "absoluteWindow.to");
366
+ if (from.getTime() > to.getTime()) {
367
+ throw new DataRuntimeError("invalid-request", "GDELT DOC absoluteWindow.from must not follow absoluteWindow.to.");
368
+ }
369
+ absoluteWindow = {
370
+ from: from.toISOString().replace(".000Z", "Z"),
371
+ to: to.toISOString().replace(".000Z", "Z"),
372
+ };
373
+ }
374
+ const timeline = TIMELINE_MODES.has(input.mode);
375
+ if (input.mode === "artlist" && input.timelineSmooth !== undefined) {
376
+ throw new DataRuntimeError("invalid-request", "timelineSmooth is accepted only for timeline modes.");
377
+ }
378
+ if (input.mode !== "artlist" && (input.maxRecords !== undefined || input.sort !== undefined)) {
379
+ throw new DataRuntimeError("invalid-request", "maxRecords and sort are accepted only for artlist mode.");
380
+ }
381
+ const domainFilters = [
382
+ ...(input.domains ?? []).map((value) => `domain:${normalizeDomain(value, "domains")}`),
383
+ ...(input.exactDomains ?? []).map((value) => `domainis:${normalizeDomain(value, "exactDomains")}`),
384
+ ];
385
+ const effectiveQueries = domainFilters.length === 0
386
+ ? [query]
387
+ : domainFilters.map((filter) => composeDomainQuery(query, filter));
388
+ for (const effectiveQuery of effectiveQueries)
389
+ lintQuery(effectiveQuery);
390
+ return {
391
+ query,
392
+ mode: input.mode,
393
+ relativeWindow,
394
+ absoluteWindow,
395
+ maxRecords: input.mode === "artlist" ? (input.maxRecords ?? 75) : null,
396
+ sort: input.mode === "artlist" ? (input.sort ?? "hybridrel") : null,
397
+ timelineSmooth: timeline ? (input.timelineSmooth ?? 0) : null,
398
+ domainFilters,
399
+ continueOnQueryError: input.continueOnQueryError ?? false,
400
+ };
401
+ }
402
+ function buildProviderQuery(query, effectiveQuery) {
403
+ return {
404
+ format: "json",
405
+ mode: query.mode,
406
+ query: effectiveQuery,
407
+ ...(query.relativeWindow
408
+ ? { TIMESPAN: `${query.relativeWindow.value}${RELATIVE_SUFFIXES[query.relativeWindow.unit]}` }
409
+ : {}),
410
+ ...(query.absoluteWindow
411
+ ? {
412
+ STARTDATETIME: gdeltDateTime(query.absoluteWindow.from),
413
+ ENDDATETIME: gdeltDateTime(query.absoluteWindow.to),
414
+ }
415
+ : {}),
416
+ ...(query.maxRecords === null ? {} : { MAXRECORDS: query.maxRecords }),
417
+ ...(query.sort === null ? {} : { sort: query.sort }),
418
+ ...(query.timelineSmooth === null ? {} : { TIMELINESMOOTH: query.timelineSmooth }),
419
+ };
420
+ }
421
+ function normalizeArticle(value, index, sourceQuery) {
422
+ const article = objectOrNull(value);
423
+ if (!article)
424
+ return null;
425
+ return {
426
+ recordIndex: index,
427
+ sourceQuery,
428
+ url: looseNullableString(article.url),
429
+ mobileUrl: looseNullableString(article.url_mobile),
430
+ title: looseString(article.title),
431
+ seenDateTime: optionalProviderDateTime(article.seendate),
432
+ socialImageUrl: looseNullableString(article.socialimage),
433
+ domain: looseString(article.domain),
434
+ language: looseString(article.language),
435
+ sourceCountry: looseString(article.sourcecountry),
436
+ };
437
+ }
438
+ function articleIdentity(article) {
439
+ return String(article.url || article.title || JSON.stringify(article));
440
+ }
441
+ function normalizeToneBins(payload, sourceQuery) {
442
+ const rawToneChart = payload.tonechart;
443
+ const bins = Array.isArray(rawToneChart)
444
+ ? rawToneChart
445
+ : objectOrNull(rawToneChart) && Array.isArray(objectOrNull(rawToneChart)?.bins)
446
+ ? objectOrNull(rawToneChart)?.bins
447
+ : [];
448
+ return bins.flatMap((value, index) => {
449
+ const bin = objectOrNull(value);
450
+ if (!bin)
451
+ return [];
452
+ const toneBin = looseString(bin.bin ?? bin.tone ?? bin.label);
453
+ const articleCount = finiteNumber(bin.count ?? bin.value);
454
+ if (!toneBin || articleCount === null || articleCount < 0)
455
+ return [];
456
+ const representativeArticles = Array.isArray(bin.articles)
457
+ ? bin.articles.flatMap((article, articleIndex) => {
458
+ const normalized = normalizeArticle(article, articleIndex, sourceQuery);
459
+ return normalized ? [normalized] : [];
460
+ })
461
+ : [];
462
+ return [
463
+ {
464
+ recordIndex: index,
465
+ sourceQuery,
466
+ toneBin,
467
+ articleCount,
468
+ representativeArticles,
469
+ },
470
+ ];
471
+ });
472
+ }
473
+ function validateModePayload(payload, mode) {
474
+ if (mode === "artlist") {
475
+ requireArray(payload.articles, "articles");
476
+ return;
477
+ }
478
+ if (mode === "tonechart") {
479
+ const chart = payload.tonechart;
480
+ if (!Array.isArray(chart) &&
481
+ !(objectOrNull(chart) && Array.isArray(objectOrNull(chart)?.bins))) {
482
+ throw providerInvalid("GDELT DOC tonechart response must contain tonechart bins.");
483
+ }
484
+ return;
485
+ }
486
+ requireArray(payload.timeline, "timeline");
487
+ }
488
+ function normalizeTimelinePoint(value, field) {
489
+ const point = requireObject(value, field);
490
+ return {
491
+ dateTime: parseProviderDateTime(point.date, `${field}.date`),
492
+ value: requireFiniteNumber(point.value, `${field}.value`),
493
+ norm: point.norm === undefined || point.norm === null
494
+ ? null
495
+ : requireFiniteNumber(point.norm, `${field}.norm`),
496
+ };
497
+ }
498
+ function normalizeQueryDetails(value) {
499
+ if (value === undefined || value === null)
500
+ return null;
501
+ const details = requireObject(value, "query_details");
502
+ const normalized = {};
503
+ for (const [key, item] of Object.entries(details)) {
504
+ if (item === null ||
505
+ typeof item === "string" ||
506
+ typeof item === "boolean" ||
507
+ (typeof item === "number" && Number.isFinite(item))) {
508
+ normalized[key] = item;
509
+ }
510
+ }
511
+ return normalized;
512
+ }
513
+ function parseProviderDateTime(value, field) {
514
+ if (typeof value !== "string")
515
+ throw providerInvalid(`${field} must be a timestamp string.`);
516
+ const match = /^(\d{4})(\d{2})(\d{2})T?(\d{2})(\d{2})(\d{2})Z$/.exec(value);
517
+ if (!match)
518
+ throw providerInvalid(`${field} must use a GDELT UTC timestamp.`);
519
+ const iso = `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}Z`;
520
+ const parsed = new Date(iso);
521
+ if (!Number.isFinite(parsed.getTime()) || parsed.toISOString().replace(".000Z", "Z") !== iso) {
522
+ throw providerInvalid(`${field} must be a valid UTC timestamp.`);
523
+ }
524
+ return iso;
525
+ }
526
+ function parseRfc3339(value, field) {
527
+ const parsed = new Date(value);
528
+ if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/.test(value) ||
529
+ !Number.isFinite(parsed.getTime()) ||
530
+ parsed.toISOString().replace(".000Z", "Z") !== value) {
531
+ throw new DataRuntimeError("invalid-request", `${field} must be an exact canonical UTC timestamp.`);
532
+ }
533
+ return parsed;
534
+ }
535
+ function gdeltDateTime(value) {
536
+ return value.replace(/[-:TZ.]/g, "").slice(0, 14);
537
+ }
538
+ function requireObject(value, field) {
539
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
540
+ throw providerInvalid(`${field} must be an object.`);
541
+ }
542
+ return value;
543
+ }
544
+ function objectOrNull(value) {
545
+ return value && typeof value === "object" && !Array.isArray(value)
546
+ ? value
547
+ : null;
548
+ }
549
+ function requireArray(value, field) {
550
+ if (!Array.isArray(value))
551
+ throw providerInvalid(`${field} must be an array.`);
552
+ return value;
553
+ }
554
+ function requireNonBlankString(value, field) {
555
+ if (typeof value !== "string" || !value.trim())
556
+ throw providerInvalid(`${field} must be a non-empty string.`);
557
+ return value.trim();
558
+ }
559
+ function looseString(value) {
560
+ return typeof value === "string" ? value.trim().replace(/\s+/g, " ") : "";
561
+ }
562
+ function looseNullableString(value) {
563
+ return looseString(value) || null;
564
+ }
565
+ function finiteNumber(value) {
566
+ if (typeof value === "number" && Number.isFinite(value))
567
+ return value;
568
+ if (typeof value === "string" && value.trim()) {
569
+ const parsed = Number(value);
570
+ return Number.isFinite(parsed) ? parsed : null;
571
+ }
572
+ return null;
573
+ }
574
+ function optionalProviderDateTime(value) {
575
+ if (value === undefined || value === null || value === "")
576
+ return null;
577
+ try {
578
+ return parseProviderDateTime(value, "article.seendate");
579
+ }
580
+ catch {
581
+ return null;
582
+ }
583
+ }
584
+ function requireFiniteNumber(value, field) {
585
+ if (typeof value !== "number" || !Number.isFinite(value))
586
+ throw providerInvalid(`${field} must be a finite number.`);
587
+ return value;
588
+ }
589
+ function providerInvalid(message) {
590
+ return new DataRuntimeError("provider-response-invalid", message);
591
+ }
592
+ function normalizeProviderFailure(error) {
593
+ return error instanceof DataRuntimeError
594
+ ? error
595
+ : providerInvalid("The GDELT DOC response could not be retrieved or normalized.");
596
+ }
597
+ function normalizeDomain(value, field) {
598
+ const normalized = value
599
+ .trim()
600
+ .toLowerCase()
601
+ .replace(/^https?:\/\//, "")
602
+ .split("/", 1)[0];
603
+ if (!/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(normalized)) {
604
+ throw new DataRuntimeError("invalid-request", `GDELT DOC ${field} values must be bare DNS domains.`);
605
+ }
606
+ return normalized;
607
+ }
608
+ function composeDomainQuery(query, domainFilter) {
609
+ const base = hasTopLevelOr(query) ? `(${query})` : query;
610
+ return `${domainFilter} ${base}`;
611
+ }
612
+ function lintQuery(query) {
613
+ const unsupported = /(?:^|\s)(site|inurl):/i.exec(query);
614
+ if (unsupported) {
615
+ throw new DataRuntimeError("invalid-request", `GDELT DOC does not support ${unsupported[1].toLowerCase()}:; use domainis: or domain: instead.`);
616
+ }
617
+ if (hasTopLevelOr(query)) {
618
+ throw new DataRuntimeError("invalid-request", "GDELT DOC boolean OR groups must be enclosed in parentheses.");
619
+ }
620
+ }
621
+ function hasTopLevelOr(query) {
622
+ let depth = 0;
623
+ let quote = null;
624
+ for (let index = 0; index < query.length; index += 1) {
625
+ const character = query[index];
626
+ if (quote) {
627
+ if (character === quote)
628
+ quote = null;
629
+ continue;
630
+ }
631
+ if (character === '"' || character === "'") {
632
+ quote = character;
633
+ continue;
634
+ }
635
+ if (character === "(") {
636
+ depth += 1;
637
+ continue;
638
+ }
639
+ if (character === ")") {
640
+ depth = Math.max(0, depth - 1);
641
+ continue;
642
+ }
643
+ if (depth === 0 &&
644
+ query.slice(index).match(/^OR\b/i) &&
645
+ (index === 0 || /\s/.test(query[index - 1]))) {
646
+ return true;
647
+ }
648
+ }
649
+ return false;
650
+ }
651
+ function discoveryWarnings(truncated) {
652
+ return [
653
+ "GDELT DOC signals are automatically extracted from uneven monitored-news coverage.",
654
+ "Returned metadata and timelines are not ground-truth facts or representative population measures.",
655
+ ...(truncated ? ["The normalized result stopped at the explicit record limit."] : []),
656
+ ];
657
+ }
658
+ //# sourceMappingURL=gdelt-doc-search.js.map