easyoref 1.14.2 → 1.15.0

package/dist/agent/extract.d.ts

@@ -0,0 +1,48 @@
+/**
+ * LLM extraction pipeline — two-tier: cheap pre-filter + expensive extraction.
+ *
+ * 1. Cheap model: single call — "which channels have important intel?"
+ * 2. Expensive model: per-post — "extract structured data"
+ * 3. Post-filter: deterministic validation on extraction results.
+ */
+import { ChatOpenAI } from "@langchain/openai";
+import type { AlertType, ChannelTracking, TrackedMessage, ValidatedExtraction } from "./types.js";
+/** Cheap model for channel pre-filtering (single call, short output) */
+export declare function getFilterLLM(): ChatOpenAI;
+/** Expensive model for structured data extraction (per-post) */
+export declare function getExtractLLM(): ChatOpenAI;
+/**
+ * Single cheap LLM call — filter channels by relevance.
+ * Returns channel names containing important military intel.
+ */
+export declare function filterChannelsCheap(tracking: ChannelTracking, alertAreas: string[], alertTs: number, alertType: AlertType): Promise<string[]>;
+/** Phase-specific extraction instructions */
+export declare function getPhaseInstructions(alertType: AlertType): string;
+export declare const EXTRACT_SYSTEM_PROMPT = "You analyze Telegram channel messages about a missile/rocket attack on Israel.\nYour job: extract factual data, assess quality, AND validate temporal relevance.\n\nCRITICAL \u2014 TIME VALIDATION:\nYou will receive the alert time and the post time. You MUST determine if this post\nis about the CURRENT attack or about a previous/different event.\n- If post discusses events clearly BEFORE the alert time \u2192 time_relevance=0\n- If post is generic military news not specific to this attack \u2192 time_relevance=0.2\n- If post discusses the current attack \u2192 time_relevance=1.0\n- If uncertain \u2192 time_relevance=0.5 (the system will use alert_history to verify)\n\nReturn ONLY valid JSON (no markdown, no explanation):\n{\n  \"region_relevance\": float,       // 0\u20131: does this message discuss the specified alert region?\n  \"source_trust\": float,           // 0\u20131: factual reporting (1.0) vs unverified rumors/panic (0.0)\n  \"tone\": \"calm\"|\"neutral\"|\"alarmist\",\n  \"time_relevance\": float,         // 0\u20131: is this post about the CURRENT attack? (see rules above)\n  \"country_origin\": string|null,   // \"Iran\",\"Yemen\",\"Lebanon\",\"Gaza\",\"Iraq\",\"Syria\" or null\n  \"rocket_count\": int|null,\n  \"is_cassette\": bool|null,\n  \"intercepted\": int|null,\n  \"intercepted_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"intercepted_qual_num\": int|null,\n  \"sea_impact\": int|null,\n  \"sea_impact_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"sea_impact_qual_num\": int|null,\n  \"open_area_impact\": int|null,\n  \"open_area_impact_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"open_area_impact_qual_num\": int|null,\n  \"hits_confirmed\": int|null,\n  \"casualties\": int|null,\n  \"injuries\": int|null,\n  \"eta_refined_minutes\": int|null,\n  \"confidence\": float\n}\n\nRules:\n- If unrelated to the alert region, set region_relevance=0 and all data fields to null.\n- If message is speculative/unconfirmed rumor, set source_trust < 0.4.\n- If message uses excessive caps, exclamation marks, panic language \u2192 tone=\"alarmist\".\n- Only extract concrete numbers explicitly stated in the text. Never guess.\n- *_qual fields: use ONLY when NO exact count is given. If exact number present, set *_qual=null.\n- \"none\" qual is only valid if explicitly stated (e.g., \"\u0432\u0441\u0435 \u043F\u0435\u0440\u0435\u0445\u0432\u0430\u0447\u0435\u043D\u044B\", \"\u043D\u0435 \u0443\u043F\u0430\u043B\u043E \u0432 \u043C\u043E\u0440\u0435\").\n- For IDF (@idf_telegram) posts about ongoing operations (not this specific attack) \u2192 time_relevance=0.\n- LANGUAGE NEUTRALITY: Posts may be in Hebrew, Russian, Arabic, or English. The language of the post\n  MUST NOT affect source_trust or confidence. Russian-language Israeli channels are equally reliable\n  and often break news faster than Hebrew ones. Judge ONLY by factual content and tone.\n- TRUST INTERCEPTION & IMPACT REPORTS: When a channel explicitly states interception results\n  (e.g., \"\u043F\u0435\u0440\u0435\u0445\u0432\u0430\u0447\u0435\u043D\u044B\", \"intercepted\", \"\u05D9\u05D9\u05E8\u05D5\u05D8\", \"\u0443\u043F\u0430\u043B\u0438 \u0432 \u043C\u043E\u0440\u0435\", \"fell in the sea\", \"\u05E0\u05E4\u05DC\u05D5 \u05D1\u05D9\u05DD\",\n  \"open area impact\", \"\u05E9\u05D8\u05D7 \u05E4\u05EA\u05D5\u05D7\"), trust these claims with source_trust >= 0.7 and confidence >= 0.7.\n  Israeli Telegram channels often report interception results before official confirmation,\n  and these reports are typically accurate. Do NOT downgrade these just because they lack official source.";
+export interface ExtractContext {
+    alertTs: number;
+    alertType: AlertType;
+    alertAreas: string[];
+    alertId: string;
+    language: string;
+}
+/**
+ * Extract structured data from posts using expensive LLM.
+ * Uses post-level dedup cache to avoid re-extracting unchanged posts.
+ */
+export declare function extractPosts(posts: TrackedMessage[], ctx: ExtractContext): Promise<ValidatedExtraction[]>;
+/**
+ * Deterministic validation of extraction results.
+ * Rejects: stale posts, irrelevant regions, untrusted sources,
+ * alarmist tone, no data, low confidence.
+ */
+export declare function postFilter(extractions: ValidatedExtraction[], alertId: string): ValidatedExtraction[];
+export declare const _test: {
+    readonly getFilterLLM: typeof getFilterLLM;
+    readonly getExtractLLM: typeof getExtractLLM;
+    readonly getPhaseInstructions: typeof getPhaseInstructions;
+    readonly EXTRACT_SYSTEM_PROMPT: "You analyze Telegram channel messages about a missile/rocket attack on Israel.\nYour job: extract factual data, assess quality, AND validate temporal relevance.\n\nCRITICAL — TIME VALIDATION:\nYou will receive the alert time and the post time. You MUST determine if this post\nis about the CURRENT attack or about a previous/different event.\n- If post discusses events clearly BEFORE the alert time → time_relevance=0\n- If post is generic military news not specific to this attack → time_relevance=0.2\n- If post discusses the current attack → time_relevance=1.0\n- If uncertain → time_relevance=0.5 (the system will use alert_history to verify)\n\nReturn ONLY valid JSON (no markdown, no explanation):\n{\n  \"region_relevance\": float,       // 0–1: does this message discuss the specified alert region?\n  \"source_trust\": float,           // 0–1: factual reporting (1.0) vs unverified rumors/panic (0.0)\n  \"tone\": \"calm\"|\"neutral\"|\"alarmist\",\n  \"time_relevance\": float,         // 0–1: is this post about the CURRENT attack? (see rules above)\n  \"country_origin\": string|null,   // \"Iran\",\"Yemen\",\"Lebanon\",\"Gaza\",\"Iraq\",\"Syria\" or null\n  \"rocket_count\": int|null,\n  \"is_cassette\": bool|null,\n  \"intercepted\": int|null,\n  \"intercepted_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"intercepted_qual_num\": int|null,\n  \"sea_impact\": int|null,\n  \"sea_impact_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"sea_impact_qual_num\": int|null,\n  \"open_area_impact\": int|null,\n  \"open_area_impact_qual\": \"all\"|\"most\"|\"many\"|\"few\"|\"exists\"|\"none\"|\"more_than\"|\"less_than\"|null,\n  \"open_area_impact_qual_num\": int|null,\n  \"hits_confirmed\": int|null,\n  \"casualties\": int|null,\n  \"injuries\": int|null,\n  \"eta_refined_minutes\": int|null,\n  \"confidence\": float\n}\n\nRules:\n- If unrelated to the alert region, set region_relevance=0 and all data fields to null.\n- If message is speculative/unconfirmed rumor, set source_trust < 0.4.\n- If message uses excessive caps, exclamation marks, panic language → tone=\"alarmist\".\n- Only extract concrete numbers explicitly stated in the text. Never guess.\n- *_qual fields: use ONLY when NO exact count is given. If exact number present, set *_qual=null.\n- \"none\" qual is only valid if explicitly stated (e.g., \"все перехвачены\", \"не упало в море\").\n- For IDF (@idf_telegram) posts about ongoing operations (not this specific attack) → time_relevance=0.\n- LANGUAGE NEUTRALITY: Posts may be in Hebrew, Russian, Arabic, or English. The language of the post\n  MUST NOT affect source_trust or confidence. Russian-language Israeli channels are equally reliable\n  and often break news faster than Hebrew ones. Judge ONLY by factual content and tone.\n- TRUST INTERCEPTION & IMPACT REPORTS: When a channel explicitly states interception results\n  (e.g., \"перехвачены\", \"intercepted\", \"יירוט\", \"упали в море\", \"fell in the sea\", \"נפלו בים\",\n  \"open area impact\", \"שטח פתוח\"), trust these claims with source_trust >= 0.7 and confidence >= 0.7.\n  Israeli Telegram channels often report interception results before official confirmation,\n  and these reports are typically accurate. Do NOT downgrade these just because they lack official source.";
+    readonly FILTER_SYSTEM_PROMPT: "You pre-filter Telegram channels for an Israeli missile alert system.\nGiven channels with their latest messages, identify which contain IMPORTANT military intel:\n- Country of origin (where rockets/missiles launched from)\n- Impact location (where they hit)\n- Warhead type / cassette munitions\n- Weapon type (ballistic, cruise, drones)\n- Damage / destruction reports\n- Interception reports (Iron Dome, David's Sling)\n- Casualty / injury reports\n\nIGNORE channels that only contain:\n- Panic, speculation, or unverified rumors\n- Rehashes of official alerts without new data\n- General commentary without actionable facts\n\nReturn ONLY valid JSON (no markdown):\n{\"relevant_channels\": [\"@channel1\", \"@channel2\"]}\nIf NO channels have important intel, return: {\"relevant_channels\": []}";
+    readonly postFilter: typeof postFilter;
+};
+//# sourceMappingURL=extract.d.ts.map

package/dist/agent/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/agent/extract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAK/C,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EAEf,cAAc,EACd,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAIpB,wEAAwE;AACxE,wBAAgB,YAAY,IAAI,UAAU,CAczC;AAED,gEAAgE;AAChE,wBAAgB,aAAa,IAAI,UAAU,CAc1C;AAuBD;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAkDnB;AAOD,6CAA6C;AAC7C,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAmBjE;AAED,eAAO,MAAM,qBAAqB,gsHAmDyE,CAAC;AAE5G,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,CAAC;IACrB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,cAAc,EAAE,EACvB,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAiJhC;AAID;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,WAAW,EAAE,mBAAmB,EAAE,EAClC,OAAO,EAAE,MAAM,GACd,mBAAmB,EAAE,CAmDvB;AAID,eAAO,MAAM,KAAK;;;;;;;CAOR,CAAC"}

package/dist/agent/extract.js

@@ -0,0 +1,375 @@
+/**
+ * LLM extraction pipeline — two-tier: cheap pre-filter + expensive extraction.
+ *
+ * 1. Cheap model: single call — "which channels have important intel?"
+ * 2. Expensive model: per-post — "extract structured data"
+ * 3. Post-filter: deterministic validation on extraction results.
+ */
+import { ChatOpenAI } from "@langchain/openai";
+import { config } from "../config.js";
+import * as logger from "../logger.js";
+import { textHash, toIsraelTime } from "./helpers.js";
+import { getCachedExtractions, saveCachedExtractions } from "./store.js";
+// ── LLM instances ──────────────────────────────────────
+/** Cheap model for channel pre-filtering (single call, short output) */
+export function getFilterLLM() {
+    return new ChatOpenAI({
+        model: config.agent.model,
+        configuration: {
+            baseURL: "https://openrouter.ai/api/v1",
+            defaultHeaders: {
+                "HTTP-Referer": "https://github.com/mikhailkogan17/EasyOref",
+                "X-Title": "EasyOref",
+            },
+        },
+        apiKey: config.agent.apiKey,
+        temperature: 0,
+        maxTokens: 200,
+    });
+}
+/** Expensive model for structured data extraction (per-post) */
+export function getExtractLLM() {
+    return new ChatOpenAI({
+        model: config.agent.extractModel,
+        configuration: {
+            baseURL: "https://openrouter.ai/api/v1",
+            defaultHeaders: {
+                "HTTP-Referer": "https://github.com/mikhailkogan17/EasyOref",
+                "X-Title": "EasyOref",
+            },
+        },
+        apiKey: config.agent.apiKey,
+        temperature: 0,
+        maxTokens: 500,
+    });
+}
+// ── Cheap pre-filter ───────────────────────────────────
+const FILTER_SYSTEM_PROMPT = `You pre-filter Telegram channels for an Israeli missile alert system.
+Given channels with their latest messages, identify which contain IMPORTANT military intel:
+- Country of origin (where rockets/missiles launched from)
+- Impact location (where they hit)
+- Warhead type / cassette munitions
+- Weapon type (ballistic, cruise, drones)
+- Damage / destruction reports
+- Interception reports (Iron Dome, David's Sling)
+- Casualty / injury reports
+
+IGNORE channels that only contain:
+- Panic, speculation, or unverified rumors
+- Rehashes of official alerts without new data
+- General commentary without actionable facts
+
+Return ONLY valid JSON (no markdown):
+{"relevant_channels": ["@channel1", "@channel2"]}
+If NO channels have important intel, return: {"relevant_channels": []}`;
+/**
+ * Single cheap LLM call — filter channels by relevance.
+ * Returns channel names containing important military intel.
+ */
+export async function filterChannelsCheap(tracking, alertAreas, alertTs, alertType) {
+    const channels = tracking.channels_with_updates;
+    if (channels.length === 0)
+        return [];
+    const channelSummaries = channels
+        .map((ch) => {
+        const posts = ch.last_tracked_messages
+            .map((m) => `  [${toIsraelTime(m.timestamp)}] ${m.text.slice(0, 200)}`)
+            .join("\n");
+        return `${ch.channel} (${ch.last_tracked_messages.length} new):\n${posts}`;
+    })
+        .join("\n\n");
+    const regionHint = alertAreas.length > 0 ? alertAreas.join(", ") : "Israel";
+    const userPrompt = `Alert: ${regionHint} at ${toIsraelTime(alertTs)}, phase: ${alertType}\n\n` + `Channels:\n${channelSummaries}`;
+    try {
+        const llm = getFilterLLM();
+        const response = await llm.invoke([
+            { role: "system", content: FILTER_SYSTEM_PROMPT },
+            { role: "user", content: userPrompt },
+        ]);
+        const raw = typeof response.content === "string"
+            ? response.content
+            : JSON.stringify(response.content);
+        const text = raw
+            .replace(/^```(?:json)?\s*\n?/i, "")
+            .replace(/\n?```\s*$/i, "");
+        const parsed = JSON.parse(text.trim());
+        logger.info("Agent: cheap pre-filter", {
+            total_channels: channels.length,
+            relevant: parsed.relevant_channels.length,
+            relevant_channels: parsed.relevant_channels,
+        });
+        return parsed.relevant_channels;
+    }
+    catch (err) {
+        logger.warn("Agent: cheap pre-filter failed, passing all channels", {
+            error: String(err),
+        });
+        // Fallback: pass all channels through
+        return channels.map((c) => c.channel);
+    }
+}
+// ── Expensive extraction ───────────────────────────────
+const QUAL_VALUES = '"all"|"most"|"many"|"few"|"exists"|"none"|"more_than"|"less_than"';
+/** Phase-specific extraction instructions */
+export function getPhaseInstructions(alertType) {
+    switch (alertType) {
+        case "early_warning":
+            return `PHASE: EARLY WARNING (radar detected launches, sirens not yet).
+Focus on: country_origin (WHERE were rockets launched from?), eta_refined_minutes, rocket_count, is_cassette.
+Do NOT extract: intercepted, sea_impact, open_area_impact, hits_confirmed, casualties, injuries — these are IMPOSSIBLE at this stage.
+If a message discusses interception results, it is about a PREVIOUS attack — set time_relevance=0.`;
+        case "siren":
+            return `PHASE: SIREN (rockets incoming, impact imminent).
+Focus on: country_origin (if not known yet), rocket_count, intercepted, sea_impact, open_area_impact, is_cassette.
+Do NOT extract: hits_confirmed, casualties, injuries — too early for confirmed damage reports.
+If a message discusses casualties or confirmed hits, verify the timing carefully - it may be about a previous attack.`;
+        case "resolved":
+            return `PHASE: RESOLVED (incident over, assessing damage).
+Focus on: intercepted (final count), hits_confirmed, casualties, injuries, open_area_impact.
+All fields are valid at this stage. Prioritize confirmed official reports.`;
+    }
+}
+export const EXTRACT_SYSTEM_PROMPT = `You analyze Telegram channel messages about a missile/rocket attack on Israel.
+Your job: extract factual data, assess quality, AND validate temporal relevance.
+
+CRITICAL — TIME VALIDATION:
+You will receive the alert time and the post time. You MUST determine if this post
+is about the CURRENT attack or about a previous/different event.
+- If post discusses events clearly BEFORE the alert time → time_relevance=0
+- If post is generic military news not specific to this attack → time_relevance=0.2
+- If post discusses the current attack → time_relevance=1.0
+- If uncertain → time_relevance=0.5 (the system will use alert_history to verify)
+
+Return ONLY valid JSON (no markdown, no explanation):
+{
+  "region_relevance": float,       // 0–1: does this message discuss the specified alert region?
+  "source_trust": float,           // 0–1: factual reporting (1.0) vs unverified rumors/panic (0.0)
+  "tone": "calm"|"neutral"|"alarmist",
+  "time_relevance": float,         // 0–1: is this post about the CURRENT attack? (see rules above)
+  "country_origin": string|null,   // "Iran","Yemen","Lebanon","Gaza","Iraq","Syria" or null
+  "rocket_count": int|null,
+  "is_cassette": bool|null,
+  "intercepted": int|null,
+  "intercepted_qual": ${QUAL_VALUES}|null,
+  "intercepted_qual_num": int|null,
+  "sea_impact": int|null,
+  "sea_impact_qual": ${QUAL_VALUES}|null,
+  "sea_impact_qual_num": int|null,
+  "open_area_impact": int|null,
+  "open_area_impact_qual": ${QUAL_VALUES}|null,
+  "open_area_impact_qual_num": int|null,
+  "hits_confirmed": int|null,
+  "casualties": int|null,
+  "injuries": int|null,
+  "eta_refined_minutes": int|null,
+  "confidence": float
+}
+
+Rules:
+- If unrelated to the alert region, set region_relevance=0 and all data fields to null.
+- If message is speculative/unconfirmed rumor, set source_trust < 0.4.
+- If message uses excessive caps, exclamation marks, panic language → tone="alarmist".
+- Only extract concrete numbers explicitly stated in the text. Never guess.
+- *_qual fields: use ONLY when NO exact count is given. If exact number present, set *_qual=null.
+- "none" qual is only valid if explicitly stated (e.g., "все перехвачены", "не упало в море").
+- For IDF (@idf_telegram) posts about ongoing operations (not this specific attack) → time_relevance=0.
+- LANGUAGE NEUTRALITY: Posts may be in Hebrew, Russian, Arabic, or English. The language of the post
+  MUST NOT affect source_trust or confidence. Russian-language Israeli channels are equally reliable
+  and often break news faster than Hebrew ones. Judge ONLY by factual content and tone.
+- TRUST INTERCEPTION & IMPACT REPORTS: When a channel explicitly states interception results
+  (e.g., "перехвачены", "intercepted", "יירוט", "упали в море", "fell in the sea", "נפלו בים",
+  "open area impact", "שטח פתוח"), trust these claims with source_trust >= 0.7 and confidence >= 0.7.
+  Israeli Telegram channels often report interception results before official confirmation,
+  and these reports are typically accurate. Do NOT downgrade these just because they lack official source.`;
+/**
+ * Extract structured data from posts using expensive LLM.
+ * Uses post-level dedup cache to avoid re-extracting unchanged posts.
+ */
+export async function extractPosts(posts, ctx) {
+    if (posts.length === 0)
+        return [];
+    // ── Post-level dedup ───────────────────────────────
+    const postHashMap = new Map();
+    for (const post of posts) {
+        const hash = textHash(post.channel + "|" + post.text.slice(0, 800));
+        postHashMap.set(hash, post);
+    }
+    const allHashes = [...postHashMap.keys()];
+    const cached = await getCachedExtractions(allHashes);
+    const cachedResults = [];
+    const newPosts = [];
+    for (const [hash, post] of postHashMap) {
+        const cachedJson = cached.get(hash);
+        if (cachedJson) {
+            cachedResults.push(JSON.parse(cachedJson));
+        }
+        else {
+            newPosts.push(post);
+        }
+    }
+    logger.info("Agent: extraction dedup", {
+        alertId: ctx.alertId,
+        total: posts.length,
+        cached: cachedResults.length,
+        new: newPosts.length,
+    });
+    if (newPosts.length === 0) {
+        return cachedResults;
+    }
+    // ── Extract new posts ──────────────────────────────
+    const llm = getExtractLLM();
+    const regionHint = ctx.alertAreas.length > 0
+        ? ctx.alertAreas.join(", ")
+        : Object.keys(config.agent.areaLabels).join(", ") || "Israel";
+    const alertTimeIL = toIsraelTime(ctx.alertTs);
+    const nowIL = toIsraelTime(Date.now());
+    const phaseInst = getPhaseInstructions(ctx.alertType);
+    const systemPrompt = EXTRACT_SYSTEM_PROMPT + "\n\n" + phaseInst;
+    const newResults = await Promise.all(newPosts.map(async (post) => {
+        const postTimeIL = toIsraelTime(post.timestamp);
+        const postAgeMin = Math.round((ctx.alertTs - post.timestamp) / 60_000);
+        const postAgeSuffix = postAgeMin > 0
+            ? `(${postAgeMin} min BEFORE alert)`
+            : postAgeMin < 0
+                ? `(${Math.abs(postAgeMin)} min AFTER alert)`
+                : "(same time as alert)";
+        const contextHeader = `Alert time: ${alertTimeIL} (Israel)\n` +
+            `Post time:  ${postTimeIL} (Israel) ${postAgeSuffix}\n` +
+            `Current time: ${nowIL} (Israel)\n` +
+            `Alert region: ${regionHint}\n` +
+            `UI language: ${ctx.language}\n`;
+        try {
+            const response = await llm.invoke([
+                { role: "system", content: systemPrompt },
+                {
+                    role: "user",
+                    content: `${contextHeader}Channel: ${post.channel}\n\nMessage:\n${post.text.slice(0, 800)}`,
+                },
+            ]);
+            const raw = typeof response.content === "string"
+                ? response.content
+                : JSON.stringify(response.content);
+            const text = raw
+                .replace(/^```(?:json)?\s*\n?/i, "")
+                .replace(/\n?```\s*$/i, "");
+            const parsed = JSON.parse(text.trim());
+            return {
+                ...parsed,
+                channel: post.channel,
+                messageUrl: post.url,
+                time_relevance: parsed.time_relevance ?? 0.5,
+                valid: true,
+            };
+        }
+        catch (err) {
+            logger.warn("Agent: extraction failed", {
+                channel: post.channel,
+                error: String(err),
+            });
+            return {
+                channel: post.channel,
+                region_relevance: 0,
+                source_trust: 0,
+                tone: "neutral",
+                time_relevance: 0,
+                country_origin: null,
+                rocket_count: null,
+                is_cassette: null,
+                intercepted: null,
+                intercepted_qual: null,
+                intercepted_qual_num: null,
+                sea_impact: null,
+                sea_impact_qual: null,
+                sea_impact_qual_num: null,
+                open_area_impact: null,
+                open_area_impact_qual: null,
+                open_area_impact_qual_num: null,
+                hits_confirmed: null,
+                casualties: null,
+                injuries: null,
+                eta_refined_minutes: null,
+                confidence: 0,
+                valid: false,
+                reject_reason: "extraction_error",
+            };
+        }
+    }));
+    // Cache new results
+    const cacheEntries = {};
+    newPosts.forEach((post, i) => {
+        const hash = textHash(post.channel + "|" + post.text.slice(0, 800));
+        cacheEntries[hash] = JSON.stringify(newResults[i]);
+    });
+    await saveCachedExtractions(cacheEntries);
+    const results = [...cachedResults, ...newResults];
+    logger.info("Agent: extracted", {
+        alertId: ctx.alertId,
+        count: results.length,
+        newLLMCalls: newResults.length,
+        cachedReused: cachedResults.length,
+    });
+    return results;
+}
+// ── Post-filter (deterministic, 0 tokens) ──────────────
+/**
+ * Deterministic validation of extraction results.
+ * Rejects: stale posts, irrelevant regions, untrusted sources,
+ * alarmist tone, no data, low confidence.
+ */
+export function postFilter(extractions, alertId) {
+    const validated = extractions.map((ext) => {
+        // V0: TIME RELEVANCE — most important check
+        if (ext.time_relevance < 0.5) {
+            return { ...ext, valid: false, reject_reason: "stale_post" };
+        }
+        // V1: region relevance
+        if (ext.region_relevance < 0.5) {
+            return { ...ext, valid: false, reject_reason: "region_irrelevant" };
+        }
+        // V2: source trust
+        if (ext.source_trust < 0.4) {
+            return { ...ext, valid: false, reject_reason: "untrusted_source" };
+        }
+        // V3: tone — reject alarmist
+        if (ext.tone === "alarmist") {
+            return { ...ext, valid: false, reject_reason: "alarmist_tone" };
+        }
+        // V4: at least one data field must be non-null
+        const hasData = ext.country_origin !== null ||
+            ext.rocket_count !== null ||
+            ext.is_cassette !== null ||
+            ext.intercepted !== null ||
+            ext.intercepted_qual !== null ||
+            ext.hits_confirmed !== null ||
+            ext.casualties !== null ||
+            ext.injuries !== null ||
+            ext.eta_refined_minutes !== null;
+        if (!hasData) {
+            return { ...ext, valid: false, reject_reason: "no_data" };
+        }
+        // V5: overall confidence floor
+        if (ext.confidence < 0.3) {
+            return { ...ext, valid: false, reject_reason: "low_confidence" };
+        }
+        return { ...ext, valid: true };
+    });
+    const passed = validated.filter((e) => e.valid);
+    const rejected = validated.filter((e) => !e.valid);
+    logger.info("Agent: post-filter", {
+        alertId,
+        passed: passed.length,
+        rejected: rejected.length,
+        reasons: rejected.map((r) => `${r.channel}:${r.reject_reason}`),
+    });
+    return validated;
+}
+// ── Exported for testing ───────────────────────────────
+export const _test = {
+    getFilterLLM,
+    getExtractLLM,
+    getPhaseInstructions,
+    EXTRACT_SYSTEM_PROMPT,
+    FILTER_SYSTEM_PROMPT,
+    postFilter,
+};
+//# sourceMappingURL=extract.js.map

package/dist/agent/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/agent/extract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AASzE,0DAA0D;AAE1D,wEAAwE;AACxE,MAAM,UAAU,YAAY;IAC1B,OAAO,IAAI,UAAU,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK;QACzB,aAAa,EAAE;YACb,OAAO,EAAE,8BAA8B;YACvC,cAAc,EAAE;gBACd,cAAc,EAAE,4CAA4C;gBAC5D,SAAS,EAAE,UAAU;aACtB;SACF;QACD,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC3B,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,GAAG;KACf,CAAC,CAAC;AACL,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa;IAC3B,OAAO,IAAI,UAAU,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;QAChC,aAAa,EAAE;YACb,OAAO,EAAE,8BAA8B;YACvC,cAAc,EAAE;gBACd,cAAc,EAAE,4CAA4C;gBAC5D,SAAS,EAAE,UAAU;aACtB;SACF;QACD,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC3B,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,GAAG;KACf,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAE1D,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;uEAiB0C,CAAC;AAExE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAyB,EACzB,UAAoB,EACpB,OAAe,EACf,SAAoB;IAEpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,gBAAgB,GAAG,QAAQ;SAC9B,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,KAAK,GAAG,EAAE,CAAC,qBAAqB;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;aACtE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,GAAG,EAAE,CAAC,OAAO,KAAK,EAAE,CAAC,qBAAqB,CAAC,MAAM,WAAW,KAAK,EAAE,CAAC;IAC7E,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE5E,MAAM,UAAU,GACd,UAAU,UAAU,OAAO,YAAY,CACrC,OAAO,CACR,YAAY,SAAS,MAAM,GAAG,cAAc,gBAAgB,EAAE,CAAC;IAElE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;YAChC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,oBAAoB,EAAE;YACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;SACtC,CAAC,CAAC;QAEH,MAAM,GAAG,GACP,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ;YAClC,CAAC,CAAC,QAAQ,CAAC,OAAO;YAClB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,GAAG;aACb,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;aACnC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAoC,CAAC;QAE1E,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACrC,cAAc,EAAE,QAAQ,CAAC,MAAM;YAC/B,QAAQ,EAAE,MAAM,CAAC,iBAAiB,CAAC,MAAM;YACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;SAC5C,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE;YAClE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;SACnB,CAAC,CAAC;QACH,sCAAsC;QACtC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,0DAA0D;AAE1D,MAAM,WAAW,GACf,mEAAmE,CAAC;AAEtE,6CAA6C;AAC7C,MAAM,UAAU,oBAAoB,CAAC,SAAoB;IACvD,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,eAAe;YAClB,OAAO;;;mGAGsF,CAAC;QAEhG,KAAK,OAAO;YACV,OAAO;;;sHAGyG,CAAC;QAEnH,KAAK,UAAU;YACb,OAAO;;2EAE8D,CAAC;IAC1E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;wBAqBb,WAAW;;;uBAGZ,WAAW;;;6BAGL,WAAW;;;;;;;;;;;;;;;;;;;;;;;;2GAwBmE,CAAC;AAU5G;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAuB,EACvB,GAAmB;IAEnB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,sDAAsD;IACtD,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpE,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAErD,MAAM,aAAa,GAA0B,EAAE,CAAC;IAChD,MAAM,QAAQ,GAAqB,EAAE,CAAC;IAEtC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAwB,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;QACrC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,GAAG,EAAE,QAAQ,CAAC,MAAM;KACrB,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,sDAAsD;IACtD,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAE5B,MAAM,UAAU,GACd,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QACvB,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;IAClE,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,YAAY,GAAG,qBAAqB,GAAG,MAAM,GAAG,SAAS,CAAC;IAEhE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAgC,EAAE;QACxD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC;QACvE,MAAM,aAAa,GACjB,UAAU,GAAG,CAAC;YACZ,CAAC,CAAC,IAAI,UAAU,oBAAoB;YACpC,CAAC,CAAC,UAAU,GAAG,CAAC;gBAChB,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,mBAAmB;gBAC7C,CAAC,CAAC,sBAAsB,CAAC;QAE7B,MAAM,aAAa,GACjB,eAAe,WAAW,aAAa;YACvC,eAAe,UAAU,aAAa,aAAa,IAAI;YACvD,iBAAiB,KAAK,aAAa;YACnC,iBAAiB,UAAU,IAAI;YAC/B,gBAAgB,GAAG,CAAC,QAAQ,IAAI,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC;gBAChC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;gBACzC;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,GAAG,aAAa,YACvB,IAAI,CAAC,OACP,iBAAiB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;iBAC3C;aACF,CAAC,CAAC;YAEH,MAAM,GAAG,GACP,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ;gBAClC,CAAC,CAAC,QAAQ,CAAC,OAAO;gBAClB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,GAAG;iBACb,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;iBACnC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAqB,CAAC;YAC3D,OAAO;gBACL,GAAG,MAAM;gBACT,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,IAAI,CAAC,GAAG;gBACpB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,GAAG;gBAC5C,KAAK,EAAE,IAAI;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;aACnB,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,gBAAgB,EAAE,CAAC;gBACnB,YAAY,EAAE,CAAC;gBACf,IAAI,EAAE,SAAkB;gBACxB,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,IAAI;gBAClB,WAAW,EAAE,IAAI;gBACjB,WAAW,EAAE,IAAI;gBACjB,gBAAgB,EAAE,IAAI;gBACtB,oBAAoB,EAAE,IAAI;gBAC1B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,IAAI;gBACrB,mBAAmB,EAAE,IAAI;gBACzB,gBAAgB,EAAE,IAAI;gBACtB,qBAAqB,EAAE,IAAI;gBAC3B,yBAAyB,EAAE,IAAI;gBAC/B,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,IAAI;gBACd,mBAAmB,EAAE,IAAI;gBACzB,UAAU,EAAE,CAAC;gBACb,KAAK,EAAE,KAAK;gBACZ,aAAa,EAAE,kBAAkB;aAClC,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,oBAAoB;IACpB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpE,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IACH,MAAM,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAE1C,MAAM,OAAO,GAAG,CAAC,GAAG,aAAa,EAAE,GAAG,UAAU,CAAC,CAAC;IAElD,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;QAC9B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,WAAW,EAAE,UAAU,CAAC,MAAM;QAC9B,YAAY,EAAE,aAAa,CAAC,MAAM;KACnC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0DAA0D;AAE1D;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,WAAkC,EAClC,OAAe;IAEf,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAuB,EAAE;QAC7D,4CAA4C;QAC5C,IAAI,GAAG,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;YAC7B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;QAC/D,CAAC;QACD,uBAAuB;QACvB,IAAI,GAAG,CAAC,gBAAgB,GAAG,GAAG,EAAE,CAAC;YAC/B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,CAAC;QACtE,CAAC;QACD,mBAAmB;QACnB,IAAI,GAAG,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC;YAC3B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,CAAC;QACrE,CAAC;QACD,6BAA6B;QAC7B,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC5B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC;QAClE,CAAC;QACD,+CAA+C;QAC/C,MAAM,OAAO,GACX,GAAG,CAAC,cAAc,KAAK,IAAI;YAC3B,GAAG,CAAC,YAAY,KAAK,IAAI;YACzB,GAAG,CAAC,WAAW,KAAK,IAAI;YACxB,GAAG,CAAC,WAAW,KAAK,IAAI;YACxB,GAAG,CAAC,gBAAgB,KAAK,IAAI;YAC7B,GAAG,CAAC,cAAc,KAAK,IAAI;YAC3B,GAAG,CAAC,UAAU,KAAK,IAAI;YACvB,GAAG,CAAC,QAAQ,KAAK,IAAI;YACrB,GAAG,CAAC,mBAAmB,KAAK,IAAI,CAAC;QACnC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;QAC5D,CAAC;QACD,+BAA+B;QAC/B,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;YACzB,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC;QACnE,CAAC;QAED,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAEnD,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;QAChC,OAAO;QACP,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,QAAQ,CAAC,MAAM;QACzB,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;KAChE,CAAC,CAAC;IAEH,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,0DAA0D;AAE1D,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,YAAY;IACZ,aAAa;IACb,oBAAoB;IACpB,qBAAqB;IACrB,oBAAoB;IACpB,UAAU;CACF,CAAC"}

package/dist/agent/filters.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Deterministic pre-filters — zero LLM tokens.
+ *
+ * Filters out noise:
+ *   - Pikud HaOref area list "простыни" (high comma count)
+ *   - Summary/recap posts with timestamp patterns "(HH:MM)", "X минут"
+ *   - IDF/Tsahal press releases (long official texts)
+ *
+ * Builds ChannelTracking structure for the LLM pipeline.
+ */
+import type { ChannelPost } from "./store.js";
+import type { ChannelTracking, TrackedMessage } from "./types.js";
+/**
+ * Pikud HaOref "простыня" — area list with many commas.
+ */
+declare function isAreaListNoise(text: string): boolean;
+/**
+ * Summary/recap posts with timestamp patterns.
+ * Real-time intel doesn't contain multiple "(HH:MM)" timestamps
+ * or "X минут/минуты" duration references.
+ */
+declare function isSummaryPost(text: string): boolean;
+/**
+ * IDF/Tsahal press releases — long official texts (>400 chars from IDF channels).
+ */
+declare function isIdfPressRelease(channel: string, text: string): boolean;
+/**
+ * Combined noise filter. Returns true if post should be filtered OUT.
+ */
+export declare function isNoise(post: ChannelPost): boolean;
+declare function toTrackedMessage(post: ChannelPost): TrackedMessage;
+/**
+ * Build ChannelTracking from session posts.
+ *
+ * Splits posts per channel into prev (already processed) and last (new).
+ * Applies deterministic noise filter on all posts.
+ * Only includes channels that have new (last) messages.
+ */
+export declare function buildChannelTracking(posts: ChannelPost[], sessionStartTs: number, lastUpdateTs: number): ChannelTracking;
+export declare const _test: {
+    readonly isAreaListNoise: typeof isAreaListNoise;
+    readonly isSummaryPost: typeof isSummaryPost;
+    readonly isIdfPressRelease: typeof isIdfPressRelease;
+    readonly isNoise: typeof isNoise;
+    readonly toTrackedMessage: typeof toTrackedMessage;
+};
+export {};
+//# sourceMappingURL=filters.d.ts.map

package/dist/agent/filters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filters.d.ts","sourceRoot":"","sources":["../../src/agent/filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EACV,eAAe,EAEf,cAAc,EACf,MAAM,YAAY,CAAC;AAQpB;;GAEG;AACH,iBAAS,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAI9C;AAED;;;;GAIG;AACH,iBAAS,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAO5C;AAED;;GAEG;AACH,iBAAS,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAGjE;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAUlD;AAID,iBAAS,gBAAgB,CAAC,IAAI,EAAE,WAAW,GAAG,cAAc,CAO3D;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,WAAW,EAAE,EACpB,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,GACnB,eAAe,CAuCjB;AAID,eAAO,MAAM,KAAK;;;;;;CAMR,CAAC"}

package/dist/agent/filters.js

@@ -0,0 +1,124 @@
+/**
+ * Deterministic pre-filters — zero LLM tokens.
+ *
+ * Filters out noise:
+ *   - Pikud HaOref area list "простыни" (high comma count)
+ *   - Summary/recap posts with timestamp patterns "(HH:MM)", "X минут"
+ *   - IDF/Tsahal press releases (long official texts)
+ *
+ * Builds ChannelTracking structure for the LLM pipeline.
+ */
+// ── Noise detectors ────────────────────────────────────
+const OREF_LINK_RE = /oref\.org\.il/i;
+const OREF_CHANNEL_RE = /pikud|פיקוד|oref/i;
+const IDF_CHANNEL_RE = /idf|צה"?ל|tsahal/i;
+/**
+ * Pikud HaOref "простыня" — area list with many commas.
+ */
+function isAreaListNoise(text) {
+    if (OREF_LINK_RE.test(text))
+        return true;
+    const commaCount = (text.match(/,/g) || []).length;
+    return commaCount >= 8;
+}
+/**
+ * Summary/recap posts with timestamp patterns.
+ * Real-time intel doesn't contain multiple "(HH:MM)" timestamps
+ * or "X минут/минуты" duration references.
+ */
+function isSummaryPost(text) {
+    // Multiple "(HH:MM)" timestamps in one post = recap/summary
+    const timeParenCount = (text.match(/\(\d{1,2}:\d{2}\)/g) || []).length;
+    if (timeParenCount >= 2)
+        return true;
+    // "X минуты" / "X минут" — Russian time duration references (recap formatting)
+    if (/\d+\s+минут[ыа]?\b/i.test(text))
+        return true;
+    return false;
+}
+/**
+ * IDF/Tsahal press releases — long official texts (>400 chars from IDF channels).
+ */
+function isIdfPressRelease(channel, text) {
+    if (!IDF_CHANNEL_RE.test(channel))
+        return false;
+    return text.length > 400;
+}
+/**
+ * Combined noise filter. Returns true if post should be filtered OUT.
+ */
+export function isNoise(post) {
+    // Pikud HaOref channels with long posts are area lists
+    if (OREF_CHANNEL_RE.test(post.channel) && post.text.length > 300)
+        return true;
+    // Area list spam (any channel)
+    if (isAreaListNoise(post.text))
+        return true;
+    // Summary/recap posts
+    if (isSummaryPost(post.text))
+        return true;
+    // IDF press releases
+    if (isIdfPressRelease(post.channel, post.text))
+        return true;
+    return false;
+}
+// ── Channel tracking structure ─────────────────────────
+function toTrackedMessage(post) {
+    return {
+        timestamp: post.ts,
+        text: post.text,
+        url: post.messageUrl,
+        channel: post.channel,
+    };
+}
+/**
+ * Build ChannelTracking from session posts.
+ *
+ * Splits posts per channel into prev (already processed) and last (new).
+ * Applies deterministic noise filter on all posts.
+ * Only includes channels that have new (last) messages.
+ */
+export function buildChannelTracking(posts, sessionStartTs, lastUpdateTs) {
+    const channelMap = new Map();
+    for (const post of posts) {
+        if (isNoise(post))
+            continue;
+        if (post.ts < sessionStartTs)
+            continue;
+        if (!channelMap.has(post.channel)) {
+            channelMap.set(post.channel, { prev: [], last: [] });
+        }
+        const bucket = channelMap.get(post.channel);
+        const msg = toTrackedMessage(post);
+        if (lastUpdateTs > 0 && post.ts <= lastUpdateTs) {
+            bucket.prev.push(msg);
+        }
+        else {
+            bucket.last.push(msg);
+        }
+    }
+    const channels_with_updates = [];
+    for (const [channel, { prev, last }] of channelMap) {
+        if (last.length > 0) {
+            channels_with_updates.push({
+                channel,
+                prev_tracked_messages: prev.sort((a, b) => a.timestamp - b.timestamp),
+                last_tracked_messages: last.sort((a, b) => a.timestamp - b.timestamp),
+            });
+        }
+    }
+    return {
+        track_start_timestamp: sessionStartTs,
+        last_update_timestamp: lastUpdateTs,
+        channels_with_updates,
+    };
+}
+// ── Exported for testing ───────────────────────────────
+export const _test = {
+    isAreaListNoise,
+    isSummaryPost,
+    isIdfPressRelease,
+    isNoise,
+    toTrackedMessage,
+};
+//# sourceMappingURL=filters.js.map

package/dist/agent/filters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filters.js","sourceRoot":"","sources":["../../src/agent/filters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AASH,0DAA0D;AAE1D,MAAM,YAAY,GAAG,gBAAgB,CAAC;AACtC,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAC5C,MAAM,cAAc,GAAG,mBAAmB,CAAC;AAE3C;;GAEG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACnD,OAAO,UAAU,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,4DAA4D;IAC5D,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACvE,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,+EAA+E;IAC/E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAClD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAe,EAAE,IAAY;IACtD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,IAAiB;IACvC,uDAAuD;IACvD,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAC9E,+BAA+B;IAC/B,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,sBAAsB;IACtB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,qBAAqB;IACrB,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0DAA0D;AAE1D,SAAS,gBAAgB,CAAC,IAAiB;IACzC,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,EAAE;QAClB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,GAAG,EAAE,IAAI,CAAC,UAAU;QACpB,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAoB,EACpB,cAAsB,EACtB,YAAoB;IAEpB,MAAM,UAAU,GAAG,IAAI,GAAG,EAGvB,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5B,IAAI,IAAI,CAAC,EAAE,GAAG,cAAc;YAAE,SAAS;QAEvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,YAAY,GAAG,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,YAAY,EAAE,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,qBAAqB,GAAyB,EAAE,CAAC;IACvD,KAAK,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QACnD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,qBAAqB,CAAC,IAAI,CAAC;gBACzB,OAAO;gBACP,qBAAqB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;gBACrE,qBAAqB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;aACtE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,qBAAqB,EAAE,cAAc;QACrC,qBAAqB,EAAE,YAAY;QACnC,qBAAqB;KACtB,CAAC;AACJ,CAAC;AAED,0DAA0D;AAE1D,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,eAAe;IACf,aAAa;IACb,iBAAiB;IACjB,OAAO;IACP,gBAAgB;CACR,CAAC"}