@tiangong-ai/cli 0.0.61 → 0.0.63

package/dist/data/connectors/gdelt-web-ngrams.js

@@ -0,0 +1,350 @@
+import { gunzipSync } from "node:zlib";
+import { DATA_MANIFEST_SCHEMA_VERSION } from "../contracts.js";
+import { DataRuntimeError } from "../runtime/errors.js";
+import { GDELT_WEB_NGRAMS_INPUT_SCHEMA, GDELT_WEB_NGRAMS_OUTPUT_SCHEMA, } from "./gdelt-web-ngrams.schemas.js";
+const BASE_URL = "https://data.gdeltproject.org";
+const PREFIX = "/gdeltv5/weblegacy/ngrams/";
+const MAX_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
+const DOCUMENTATION = "https://blog.gdeltproject.org/using-the-new-web-ngrams-dataset-to-find-relevant-coverage/";
+export const gdeltWebNgramsConnector = {
+    schemaVersion: DATA_MANIFEST_SCHEMA_VERSION,
+    capabilityId: "gdelt.web-ngrams",
+    capabilityVersion: "1.0.0",
+    minimumCliVersion: "0.0.61",
+    provider: { providerId: "gdelt", name: "GDELT Project" },
+    sourceCategory: "global-news-metadata",
+    endpoints: [
+        {
+            endpointId: "gdelt-web-ngrams",
+            baseUrl: BASE_URL,
+            pathPrefixes: [PREFIX],
+            allowedMethods: ["GET"],
+            allowedContentTypes: [
+                "application/json",
+                "text/plain",
+                "application/gzip",
+                "application/x-gzip",
+                "application/octet-stream",
+            ],
+        },
+    ],
+    license: {
+        name: "GDELT Project data",
+        url: "https://www.gdeltproject.org/about.html",
+        restrictions: [
+            "Non-consumptive quadgram histograms and link metadata, not article full text.",
+            "Source-specific rights still apply when following or reusing linked content.",
+        ],
+    },
+    credentials: [],
+    limits: {
+        timeoutMs: 60_000,
+        maxRequestBytes: 4_096,
+        maxResponseBytes: 20 * 1024 * 1024,
+        maxPages: 2,
+        maxRecords: 5_000,
+        maxRetries: 2,
+        maxRetryDelayMs: 60_000,
+        maxRedirects: 2,
+    },
+    diagnostics: { static: true, live: false },
+    freshness: {
+        kind: "provider-current",
+        description: "Explicit provider minute files; nominal publication lag and missing minutes do not establish continuous coverage.",
+    },
+    limitations: [
+        "One explicit paired minute per call; at most 64 MiB decompressed per file. Missing or corrupt files are blocked, never an empty successful search.",
+        "Literal 1–4-word phrases only. No DOC query-language equivalence, tone timeline, ranking, translation, or complete corpus guarantee.",
+        "DOCIDs reset per minute; use fileTimestamp plus documentId. Repeated URLs across minutes are not silently deduplicated.",
+    ],
+    discovery: {
+        source: {
+            maintainedBy: "The GDELT Project",
+            summary: "Web NGrams quadgram histograms and paired article tables of contents.",
+            description: "A file-based keyword-search route recommended during legacy search infrastructure migration. The official paired files associate per-document quadgrams with URLs and titles without article bodies.",
+            coverage: {
+                geographic: "Monitored global web news with uneven source and language coverage.",
+                temporal: "Explicit published UTC minutes. Legacy 15-minute processing can leave missing minutes; an arbitrary time range is not guaranteed covered.",
+                granularity: "Per-minute, per-document quadgram presence joined to the same minute's TOC.",
+            },
+        },
+        summary: "Find article links containing literal words or short phrases in one published minute.",
+        description: "Fetch both official GZIP files, validate and join file-scoped document IDs, and scan every valid quadgram before returning bounded matching article metadata.",
+        provides: [
+            "Literal case-insensitive word/phrase matching with any/all document selection and optional TOC language filters.",
+            "Matched phrases, file provenance, validation counts, and explicit omitted-result counts.",
+        ],
+        doesNotProvide: [
+            "DOC operators, DOC sentiment/volume timelines, semantic search, or relevance ranking.",
+            "Article bodies, reconstructed text, reliable phrase-frequency sums, or all-news coverage.",
+        ],
+        selectionHints: [
+            "Choose for literal topic discovery when a published minute is known, including when DOC is load-shedding. It does not automatically replace a DOC query.",
+            "Choose GKG for machine-coded themes/entities/tone, Events for coded events, and Mentions for event-document linkage. Join only when the research question needs those semantics.",
+            "For ranges, the caller must explicitly enumerate intended minutes and report missing minutes; a successful single minute is not a complete range.",
+        ],
+        typicalUseCases: ["Find candidate article URLs containing a disease name in a known minute."],
+        sourceDocumentation: [{ title: "Using the new Web NGrams dataset", url: DOCUMENTATION }],
+    },
+    operations: [
+        {
+            operationId: "search",
+            operationVersion: "1.0.0",
+            summary: "Search one explicit NGrams/TOC minute pair.",
+            description: "Validate both files and join document-scoped literal matches to metadata; report malformed rows and output truncation separately from no matches.",
+            inputSchema: GDELT_WEB_NGRAMS_INPUT_SCHEMA,
+            outputSchema: GDELT_WEB_NGRAMS_OUTPUT_SCHEMA,
+            execute: search,
+        },
+    ],
+};
+async function search(context) {
+    const input = context.input;
+    const parsedDate = new Date(input.fileTimestamp);
+    const phrases = input.phrases.map((phrase) => phrase.trim().replace(/\s+/gu, " "));
+    if (!Number.isFinite(parsedDate.getTime()) ||
+        parsedDate.toISOString() !== input.fileTimestamp.replace("Z", ".000Z") ||
+        phrases.some((phrase) => !phrase || phrase.split(" ").length > 4 || !/[\p{L}\p{N}]/u.test(phrase)) ||
+        new Set(phrases.map((phrase) => phrase.toLowerCase())).size !== phrases.length ||
+        context.limits.maxPages < 2) {
+        throw new DataRuntimeError("invalid-request", "Use an actual UTC minute, unique literal 1–4-word phrases, and a request budget for both files.");
+    }
+    const stem = input.fileTimestamp.replace(/[-:TZ]/gu, "");
+    const observations = [];
+    const files = [];
+    async function read(kind) {
+        const path = `${PREFIX}${stem}.${kind === "toc" ? "toc.json" : "ngrams.txt"}.gz`;
+        const response = await context.http.request({
+            endpointId: "gdelt-web-ngrams",
+            method: "GET",
+            path,
+        });
+        let bytes;
+        let content;
+        try {
+            bytes = gunzipSync(response.bytes, { maxOutputLength: MAX_UNCOMPRESSED_BYTES });
+            content = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        }
+        catch {
+            throw new DataRuntimeError("provider-response-invalid", "GDELT minute file is not valid bounded UTF-8 GZIP data.", {
+                details: {
+                    kind,
+                    fileTimestamp: input.fileTimestamp,
+                    maxUncompressedBytes: MAX_UNCOMPRESSED_BYTES,
+                },
+            });
+        }
+        observations.push(response.observation);
+        files.push({
+            kind,
+            sourceUrl: BASE_URL + path,
+            compressedBytes: response.bytes.byteLength,
+            uncompressedBytes: bytes.byteLength,
+            responseDigest: response.observation.responseDigest,
+        });
+        return content;
+    }
+    const statistics = {
+        tocRows: 0,
+        ngramRows: 0,
+        invalidTocRows: 0,
+        invalidNgramRows: 0,
+        orphanNgramRows: 0,
+        ambiguousDocumentCount: 0,
+        duplicateTocRows: 0,
+    };
+    const articles = new Map();
+    const ambiguous = new Set();
+    for (const line of lines(await read("toc"))) {
+        statistics.tocRows++;
+        const article = parseArticle(line, input.fileTimestamp);
+        if (!article) {
+            statistics.invalidTocRows++;
+            continue;
+        }
+        if (ambiguous.has(article.documentId)) {
+            statistics.duplicateTocRows++;
+            continue;
+        }
+        const previous = articles.get(article.documentId);
+        if (previous) {
+            statistics.duplicateTocRows++;
+            if (JSON.stringify(previous) !== JSON.stringify(article)) {
+                articles.delete(article.documentId);
+                ambiguous.add(article.documentId);
+            }
+        }
+        else
+            articles.set(article.documentId, article);
+    }
+    statistics.ambiguousDocumentCount = ambiguous.size;
+    const patterns = phrases.map((phrase) => new RegExp(`(?<![\\p{L}\\p{M}\\p{N}_])${phrase.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")}(?![\\p{L}\\p{M}\\p{N}_])`, "iu"));
+    const matches = new Map();
+    for (const line of lines(await read("ngrams"))) {
+        statistics.ngramRows++;
+        const fields = line.split("\t");
+        const id = Number(fields[0]);
+        const count = Number(fields[2]);
+        if (fields.length !== 3 ||
+            !/^\d+$/u.test(fields[0]) ||
+            !Number.isSafeInteger(id) ||
+            !fields[1].trim() ||
+            !/^\d+$/u.test(fields[2]) ||
+            !Number.isSafeInteger(count) ||
+            count < 1) {
+            statistics.invalidNgramRows++;
+            continue;
+        }
+        const article = articles.get(id);
+        if (!article) {
+            statistics.orphanNgramRows++;
+            continue;
+        }
+        if (input.languages && (!article.language || !input.languages.includes(article.language)))
+            continue;
+        const gram = fields[1].trim().replace(/\s+/gu, " ");
+        patterns.forEach((pattern, index) => {
+            if (pattern.test(gram)) {
+                const found = matches.get(id) ?? new Set();
+                found.add(index);
+                matches.set(id, found);
+            }
+        });
+    }
+    if (!statistics.tocRows || !statistics.ngramRows) {
+        throw new DataRuntimeError("provider-response-invalid", "A published GDELT pair contains an empty file; no searchable coverage is established.");
+    }
+    const match = input.match ?? "any";
+    const matched = [...matches.entries()]
+        .filter(([, found]) => match === "any" || found.size === phrases.length)
+        .sort(([a], [b]) => a - b);
+    const records = matched.slice(0, context.limits.maxRecords).map(([id, found]) => ({
+        ...articles.get(id),
+        matchedPhrases: phrases.filter((_, index) => found.has(index)),
+    }));
+    const omittedRecordCount = matched.length - records.length;
+    const invalid = statistics.invalidTocRows +
+        statistics.invalidNgramRows +
+        statistics.orphanNgramRows +
+        statistics.ambiguousDocumentCount >
+        0;
+    const partial = invalid || omittedRecordCount > 0;
+    return {
+        status: partial ? "partial" : "success",
+        data: {
+            fileTimestamp: input.fileTimestamp,
+            phrases,
+            match,
+            languages: input.languages ?? [],
+            records,
+            matchedDocumentCount: matched.length,
+            omittedRecordCount,
+            statistics,
+            files,
+            stopReason: omittedRecordCount
+                ? "record-limit"
+                : invalid
+                    ? "invalid-source-rows"
+                    : "file-complete",
+        },
+        summary: {
+            recordCount: records.length,
+            pageCount: 2,
+            chunkCount: 1,
+            truncated: omittedRecordCount > 0,
+            completeness: partial ? "partial" : "complete",
+            ...(partial
+                ? {
+                    missing: [
+                        {
+                            kind: "range",
+                            identifiers: [
+                                ...(invalid
+                                    ? [`${input.fileTimestamp}:unvalidated-or-unlinked-source-rows`]
+                                    : []),
+                                ...(omittedRecordCount
+                                    ? [
+                                        `${input.fileTimestamp}:matching-document-offsets:${records.length}-${matched.length - 1}`,
+                                    ]
+                                    : []),
+                            ],
+                        },
+                    ],
+                }
+                : {}),
+        },
+        warnings: [
+            ...(invalid
+                ? [
+                    "Invalid or ambiguously linked source rows prevent complete matching coverage; inspect statistics. Match counts refer only to valid linked rows.",
+                ]
+                : []),
+            ...(omittedRecordCount
+                ? [
+                    `The complete file scan found ${matched.length} matching documents; ${omittedRecordCount} are omitted by maxRecords. Rerun with a sufficient allowed record limit or explicitly partition the query.`,
+                ]
+                : []),
+        ],
+        errors: partial
+            ? [
+                {
+                    code: "partial-result",
+                    message: "GDELT matching coverage or returned records are incomplete; inspect statistics and omittedRecordCount.",
+                    retryable: false,
+                    userActionRequired: false,
+                    details: { ...statistics, omittedRecordCount },
+                },
+            ]
+            : [],
+        observations,
+    };
+}
+function* lines(content) {
+    for (const match of content.matchAll(/[^\r\n]+/gu))
+        if (match[0].trim())
+            yield match[0];
+}
+function validUrl(value) {
+    if (typeof value !== "string")
+        return false;
+    try {
+        const url = new URL(value);
+        return ["http:", "https:"].includes(url.protocol) && !url.username && !url.password;
+    }
+    catch {
+        return false;
+    }
+}
+function parseArticle(line, fileTimestamp) {
+    try {
+        const row = JSON.parse(line);
+        if (!row ||
+            Array.isArray(row) ||
+            typeof row !== "object" ||
+            !Number.isSafeInteger(row.ID) ||
+            row.ID < 0 ||
+            !validUrl(row.url))
+            return null;
+        for (const field of ["title", "lang", "date", "img"])
+            if (row[field] != null && typeof row[field] !== "string")
+                return null;
+        if (typeof row.date === "string" &&
+            (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/u.test(row.date) ||
+                !Number.isFinite(Date.parse(row.date)) ||
+                new Date(row.date).toISOString() !== row.date.replace(/(?<!\.\d{3})Z$/u, ".000Z")))
+            return null;
+        return {
+            fileTimestamp,
+            documentId: row.ID,
+            url: row.url,
+            title: row.title ?? null,
+            language: row.lang ?? null,
+            date: row.date ?? null,
+            imageReference: row.img ?? null,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=gdelt-web-ngrams.js.map

package/dist/data/connectors/gdelt-web-ngrams.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gdelt-web-ngrams.js","sourceRoot":"","sources":["../../../src/data/connectors/gdelt-web-ngrams.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAOvC,OAAO,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,+BAA+B,CAAC;AAEvC,MAAM,QAAQ,GAAG,+BAA+B,CAAC;AACjD,MAAM,MAAM,GAAG,4BAA4B,CAAC;AAC5C,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAChD,MAAM,aAAa,GACjB,2FAA2F,CAAC;AAiB9F,MAAM,CAAC,MAAM,uBAAuB,GAA4B;IAC9D,aAAa,EAAE,4BAA4B;IAC3C,YAAY,EAAE,kBAAkB;IAChC,iBAAiB,EAAE,OAAO;IAC1B,iBAAiB,EAAE,QAAQ;IAC3B,QAAQ,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE;IACxD,cAAc,EAAE,sBAAsB;IACtC,SAAS,EAAE;QACT;YACE,UAAU,EAAE,kBAAkB;YAC9B,OAAO,EAAE,QAAQ;YACjB,YAAY,EAAE,CAAC,MAAM,CAAC;YACtB,cAAc,EAAE,CAAC,KAAK,CAAC;YACvB,mBAAmB,EAAE;gBACnB,kBAAkB;gBAClB,YAAY;gBACZ,kBAAkB;gBAClB,oBAAoB;gBACpB,0BAA0B;aAC3B;SACF;KACF;IACD,OAAO,EAAE;QACP,IAAI,EAAE,oBAAoB;QAC1B,GAAG,EAAE,yCAAyC;QAC9C,YAAY,EAAE;YACZ,+EAA+E;YAC/E,8EAA8E;SAC/E;KACF;IACD,WAAW,EAAE,EAAE;IACf,MAAM,EAAE;QACN,SAAS,EAAE,MAAM;QACjB,eAAe,EAAE,KAAK;QACtB,gBAAgB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAClC,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,KAAK;QACjB,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,MAAM;QACvB,YAAY,EAAE,CAAC;KAChB;IACD,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE;IAC1C,SAAS,EAAE;QACT,IAAI,EAAE,kBAAkB;QACxB,WAAW,EACT,mHAAmH;KACtH;IACD,WAAW,EAAE;QACX,oJAAoJ;QACpJ,sIAAsI;QACtI,yHAAyH;KAC1H;IACD,SAAS,EAAE;QACT,MAAM,EAAE;YACN,YAAY,EAAE,mBAAmB;YACjC,OAAO,EAAE,uEAAuE;YAChF,WAAW,EACT,sMAAsM;YACxM,QAAQ,EAAE;gBACR,UAAU,EAAE,qEAAqE;gBACjF,QAAQ,EACN,2IAA2I;gBAC7I,WAAW,EAAE,6EAA6E;aAC3F;SACF;QACD,OAAO,EACL,uFAAuF;QACzF,WAAW,EACT,+JAA+J;QACjK,QAAQ,EAAE;YACR,kHAAkH;YAClH,0FAA0F;SAC3F;QACD,cAAc,EAAE;YACd,uFAAuF;YACvF,2FAA2F;SAC5F;QACD,cAAc,EAAE;YACd,0JAA0J;YAC1J,kLAAkL;YAClL,mJAAmJ;SACpJ;QACD,eAAe,EAAE,CAAC,0EAA0E,CAAC;QAC7F,mBAAmB,EAAE,CAAC,EAAE,KAAK,EAAE,kCAAkC,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC;KACzF;IACD,UAAU,EAAE;QACV;YACE,WAAW,EAAE,QAAQ;YACrB,gBAAgB,EAAE,OAAO;YACzB,OAAO,EAAE,6CAA6C;YACtD,WAAW,EACT,mJAAmJ;YACrJ,WAAW,EAAE,6BAA6B;YAC1C,YAAY,EAAE,8BAA8B;YAC5C,OAAO,EAAE,MAAM;SAChB;KACF;CACF,CAAC;AAEF,KAAK,UAAU,MAAM,CAAC,OAAsC;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAc,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACnF,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACtC,UAAU,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC;QACtE,OAAO,CAAC,IAAI,CACV,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CACrF;QACD,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM;QAC9E,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,EAC3B,CAAC;QACD,MAAM,IAAI,gBAAgB,CACxB,iBAAiB,EACjB,iGAAiG,CAClG,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,MAAM,KAAK,GAMN,EAAE,CAAC;IACR,KAAK,UAAU,IAAI,CAAC,IAAsB;QACxC,MAAM,IAAI,GAAG,GAAG,MAAM,GAAG,IAAI,IAAI,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC;QACjF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC1C,UAAU,EAAE,kBAAkB;YAC9B,MAAM,EAAE,KAAK;YACb,IAAI;SACL,CAAC,CAAC;QACH,IAAI,KAAa,CAAC;QAClB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,sBAAsB,EAAE,CAAC,CAAC;YAChF,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,gBAAgB,CACxB,2BAA2B,EAC3B,yDAAyD,EACzD;gBACE,OAAO,EAAE;oBACP,IAAI;oBACJ,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,oBAAoB,EAAE,sBAAsB;iBAC7C;aACF,CACF,CAAC;QACJ,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI;YACJ,SAAS,EAAE,QAAQ,GAAG,IAAI;YAC1B,eAAe,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU;YAC1C,iBAAiB,EAAE,KAAK,CAAC,UAAU;YACnC,cAAc,EAAE,QAAQ,CAAC,WAAW,CAAC,cAAc;SACpD,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,UAAU,GAAG;QACjB,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,cAAc,EAAE,CAAC;QACjB,gBAAgB,EAAE,CAAC;QACnB,eAAe,EAAE,CAAC;QAClB,sBAAsB,EAAE,CAAC;QACzB,gBAAgB,EAAE,CAAC;KACpB,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5C,UAAU,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,UAAU,CAAC,cAAc,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QACD,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YACtC,UAAU,CAAC,gBAAgB,EAAE,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzD,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACpC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;;YAAM,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;IACD,UAAU,CAAC,sBAAsB,GAAG,SAAS,CAAC,IAAI,CAAC;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAC1B,CAAC,MAAM,EAAE,EAAE,CACT,IAAI,MAAM,CACR,6BAA6B,MAAM,CAAC,OAAO,CAAC,sBAAsB,EAAE,MAAM,CAAC,2BAA2B,EACtG,IAAI,CACL,CACJ,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC/C,UAAU,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,IACE,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;YAC1B,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YACzB,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE;YAClB,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;YAC1B,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC;YAC5B,KAAK,GAAG,CAAC,EACT,CAAC;YACD,UAAU,CAAC,gBAAgB,EAAE,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,UAAU,CAAC,eAAe,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvF,SAAS;QACX,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACrD,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;gBACnD,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QACjD,MAAM,IAAI,gBAAgB,CACxB,2BAA2B,EAC3B,uFAAuF,CACxF,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC;IACnC,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SACnC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC;SACvE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAChF,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAE;QACpB,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KAC/D,CAAC,CAAC,CAAC;IACJ,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3D,MAAM,OAAO,GACX,UAAU,CAAC,cAAc;QACvB,UAAU,CAAC,gBAAgB;QAC3B,UAAU,CAAC,eAAe;QAC1B,UAAU,CAAC,sBAAsB;QACnC,CAAC,CAAC;IACJ,MAAM,OAAO,GAAG,OAAO,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAClD,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QACvC,IAAI,EAAE;YACJ,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO;YACP,KAAK;YACL,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;YAChC,OAAO;YACP,oBAAoB,EAAE,OAAO,CAAC,MAAM;YACpC,kBAAkB;YAClB,UAAU;YACV,KAAK;YACL,UAAU,EAAE,kBAAkB;gBAC5B,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,OAAO;oBACP,CAAC,CAAC,qBAAqB;oBACvB,CAAC,CAAC,eAAe;SACtB;QACD,OAAO,EAAE;YACP,WAAW,EAAE,OAAO,CAAC,MAAM;YAC3B,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,kBAAkB,GAAG,CAAC;YACjC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;YAC9C,GAAG,CAAC,OAAO;gBACT,CAAC,CAAC;oBACE,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,OAAgB;4BACtB,WAAW,EAAE;gCACX,GAAG,CAAC,OAAO;oCACT,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,aAAa,sCAAsC,CAAC;oCAChE,CAAC,CAAC,EAAE,CAAC;gCACP,GAAG,CAAC,kBAAkB;oCACpB,CAAC,CAAC;wCACE,GAAG,KAAK,CAAC,aAAa,8BAA8B,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;qCAC3F;oCACH,CAAC,CAAC,EAAE,CAAC;6BACR;yBACF;qBACF;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;SACR;QACD,QAAQ,EAAE;YACR,GAAG,CAAC,OAAO;gBACT,CAAC,CAAC;oBACE,iJAAiJ;iBAClJ;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,kBAAkB;gBACpB,CAAC,CAAC;oBACE,gCAAgC,OAAO,CAAC,MAAM,wBAAwB,kBAAkB,6GAA6G;iBACtM;gBACH,CAAC,CAAC,EAAE,CAAC;SACR;QACD,MAAM,EAAE,OAAO;YACb,CAAC,CAAC;gBACE;oBACE,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EACL,wGAAwG;oBAC1G,SAAS,EAAE,KAAK;oBAChB,kBAAkB,EAAE,KAAK;oBACzB,OAAO,EAAE,EAAE,GAAG,UAAU,EAAE,kBAAkB,EAAE;iBAC/C;aACF;YACH,CAAC,CAAC,EAAE;QACN,YAAY;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,CAAC,KAAK,CAAC,OAAe;IAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;YAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;AAC1F,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,aAAqB;IACvD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmC,CAAC;QAC/D,IACE,CAAC,GAAG;YACJ,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAClB,OAAO,GAAG,KAAK,QAAQ;YACvB,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,GAAG,CAAC,EAAa,GAAG,CAAC;YACtB,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAElB,OAAO,IAAI,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;YAClD,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;QACxE,IACE,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;YAC5B,CAAC,CAAC,qDAAqD,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBACpE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;YAEpF,OAAO,IAAI,CAAC;QACd,OAAO;YACL,aAAa;YACb,UAAU,EAAE,GAAG,CAAC,EAAY;YAC5B,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,KAAK,EAAG,GAAG,CAAC,KAA4B,IAAI,IAAI;YAChD,QAAQ,EAAG,GAAG,CAAC,IAA2B,IAAI,IAAI;YAClD,IAAI,EAAG,GAAG,CAAC,IAA2B,IAAI,IAAI;YAC9C,cAAc,EAAG,GAAG,CAAC,GAA0B,IAAI,IAAI;SACxD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/data/connectors/gdelt-web-ngrams.schemas.d.ts

@@ -0,0 +1,188 @@
+export declare const GDELT_WEB_NGRAMS_INPUT_SCHEMA: {
+    readonly $schema: "https://json-schema.org/draft/2020-12/schema";
+    readonly $id: "https://schemas.tiangong.ai/data/gdelt/web-ngrams-search-input.v1.json";
+    readonly type: "object";
+    readonly additionalProperties: false;
+    readonly required: readonly ["fileTimestamp", "phrases"];
+    readonly properties: {
+        readonly fileTimestamp: {
+            readonly type: "string";
+            readonly pattern: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:00Z$";
+            readonly description: "One explicit UTC minute identifying a paired NGrams/TOC file, not an article publication-time filter. Missing minutes are possible; no implicit time-window expansion or latest polling.";
+            readonly examples: readonly ["2026-06-30T20:16:00Z"];
+        };
+        readonly phrases: {
+            readonly type: "array";
+            readonly minItems: 1;
+            readonly maxItems: 10;
+            readonly uniqueItems: true;
+            readonly description: "Literal phrases, each 1–4 whitespace-separated words. Matches ignore case and repeated whitespace, use Unicode letter/number word boundaries, and otherwise preserve punctuation. No stemming, translation, DOC operators, regex, or full-text reconstruction.";
+            readonly examples: readonly [readonly ["climate change", "disease"]];
+            readonly items: {
+                readonly type: "string";
+                readonly minLength: 1;
+                readonly maxLength: 200;
+            };
+        };
+        readonly match: {
+            readonly enum: readonly ["any", "all"];
+            readonly default: "any";
+            readonly description: "Whether any or all phrases must occur in the same document; all may match different quadgrams. Returned matches indicate presence, not occurrence counts.";
+            readonly examples: readonly ["any"];
+        };
+        readonly languages: {
+            readonly type: "array";
+            readonly minItems: 1;
+            readonly maxItems: 20;
+            readonly uniqueItems: true;
+            readonly description: "Optional exact TOC language-code filter. Missing language metadata cannot satisfy a filter. This is not a query-translation instruction.";
+            readonly examples: readonly [readonly ["en", "fr"]];
+            readonly items: {
+                readonly type: "string";
+                readonly pattern: "^[a-z]{2,3}$";
+            };
+        };
+    };
+};
+export declare const GDELT_WEB_NGRAMS_OUTPUT_SCHEMA: {
+    readonly $schema: "https://json-schema.org/draft/2020-12/schema";
+    readonly $id: "https://schemas.tiangong.ai/data/gdelt/web-ngrams-search-output.v1.json";
+    readonly type: "object";
+    readonly additionalProperties: false;
+    readonly required: readonly ["fileTimestamp", "phrases", "match", "languages", "records", "matchedDocumentCount", "omittedRecordCount", "statistics", "files", "stopReason"];
+    readonly properties: {
+        readonly fileTimestamp: {
+            readonly type: "string";
+        };
+        readonly phrases: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+        };
+        readonly match: {
+            readonly enum: readonly ["any", "all"];
+        };
+        readonly languages: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+        };
+        readonly matchedDocumentCount: {
+            readonly type: "integer";
+            readonly minimum: 0;
+        };
+        readonly omittedRecordCount: {
+            readonly type: "integer";
+            readonly minimum: 0;
+        };
+        readonly stopReason: {
+            readonly enum: readonly ["file-complete", "invalid-source-rows", "record-limit"];
+        };
+        readonly statistics: {
+            readonly type: "object";
+            readonly additionalProperties: false;
+            readonly required: readonly ["tocRows", "ngramRows", "invalidTocRows", "invalidNgramRows", "orphanNgramRows", "ambiguousDocumentCount", "duplicateTocRows"];
+            readonly properties: {
+                readonly tocRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly ngramRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly invalidTocRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly invalidNgramRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly orphanNgramRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly ambiguousDocumentCount: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+                readonly duplicateTocRows: {
+                    readonly type: "integer";
+                    readonly minimum: 0;
+                };
+            };
+        };
+        readonly files: {
+            readonly type: "array";
+            readonly minItems: 2;
+            readonly maxItems: 2;
+            readonly items: {
+                readonly type: "object";
+                readonly additionalProperties: false;
+                readonly required: readonly ["kind", "sourceUrl", "compressedBytes", "uncompressedBytes", "responseDigest"];
+                readonly properties: {
+                    readonly kind: {
+                        readonly enum: readonly ["toc", "ngrams"];
+                    };
+                    readonly sourceUrl: {
+                        readonly type: "string";
+                    };
+                    readonly compressedBytes: {
+                        readonly type: "integer";
+                        readonly minimum: 0;
+                    };
+                    readonly uncompressedBytes: {
+                        readonly type: "integer";
+                        readonly minimum: 0;
+                    };
+                    readonly responseDigest: {
+                        readonly type: "string";
+                    };
+                };
+            };
+        };
+        readonly records: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly additionalProperties: false;
+                readonly required: readonly ["fileTimestamp", "documentId", "url", "title", "language", "date", "imageReference", "matchedPhrases"];
+                readonly properties: {
+                    readonly fileTimestamp: {
+                        readonly type: "string";
+                    };
+                    readonly documentId: {
+                        readonly type: "integer";
+                        readonly minimum: 0;
+                    };
+                    readonly url: {
+                        readonly type: "string";
+                    };
+                    readonly title: {
+                        readonly type: readonly ["string", "null"];
+                    };
+                    readonly language: {
+                        readonly type: readonly ["string", "null"];
+                    };
+                    readonly date: {
+                        readonly type: readonly ["string", "null"];
+                    };
+                    readonly imageReference: {
+                        readonly type: readonly ["string", "null"];
+                        readonly description: "Unvalidated provider image-reference text, preserved verbatim; it may not be a usable URL. Never fetched or rendered by this operation.";
+                    };
+                    readonly matchedPhrases: {
+                        readonly type: "array";
+                        readonly minItems: 1;
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                    };
+                };
+            };
+        };
+    };
+};