@deepcitation/deepcitation-js 1.1.27 → 1.1.28

package/lib/parsing/normalizeCitation.js

@@ -1,198 +0,0 @@
-export const removeCitations = (pageText, leaveKeySpanBehind) => {
-    const citationRegex = /<cite\s+(?:fileId|attachmentId)='(\w{0,25})'\s+start_page[\_a-zA-Z]*='page[\_a-zA-Z]*(\d+)_index_(\d+)'\s+full_phrase='((?:[^'\\]|\\.)*)'\s+key_span='((?:[^'\\]|\\.)*)'\s+line(?:_ids|Ids)='([^']+)'(?:\s+(value|reasoning)='((?:[^'\\]|\\.)*)')?\s*\/>/g;
-    return pageText.replace(citationRegex, (match, attachmentId, pageNumber, index, fullPhrase, keySpan, lineIds, value) => {
-        //it is still value= so we need to remove the value=
-        if (leaveKeySpanBehind) {
-            return keySpan?.replace(/key_span=['"]|['"]/g, "") || "";
-        }
-        else {
-            return "";
-        }
-    });
-};
-export const removePageNumberMetadata = (pageText) => {
-    return pageText
-        .replace(/<page_number_\d+_index_\d+>/g, "")
-        .replace(/<\/page_number_\d+_index_\d+>/g, "")
-        .trim();
-};
-export const removeLineIdMetadata = (pageText) => {
-    const lineIdRegex = /<line id="[^"]*">|<\/line>/g;
-    return pageText.replace(lineIdRegex, "");
-};
-export const getCitationPageNumber = (startPageKey) => {
-    //page_number_{page_number}_index_{page_index} or page_number_{page_number} or page_key_{page_number}_index_{page_index}
-    if (!startPageKey)
-        return null;
-    //regex first \d+ is the page number
-    const pageNumber = startPageKey.match(/\d+/)?.[0];
-    return pageNumber ? parseInt(pageNumber) : null;
-};
-export const normalizeCitations = (response) => {
-    let trimmedResponse = response?.trim() || "";
-    const citationParts = trimmedResponse.split(/(<cite[\s\S]*?(?:\/>|<\/cite>))/gm);
-    if (citationParts.length <= 1) {
-        return normalizeCitationContent(trimmedResponse);
-    }
-    trimmedResponse = citationParts
-        .map((part) => part.startsWith("<cite") ? normalizeCitationContent(part) : part)
-        .join("");
-    return trimmedResponse;
-};
-const normalizeCitationContent = (input) => {
-    let normalized = input;
-    // 1. Standardize self-closing tags
-    // Replace ></cite> with /> for consistency
-    normalized = normalized.replace(/><\/cite>/g, "/>");
-    const canonicalizeCiteAttributeKey = (key) => {
-        const lowerKey = key.toLowerCase();
-        if (lowerKey === "fullphrase" || lowerKey === "full_phrase")
-            return "full_phrase";
-        if (lowerKey === "lineids" || lowerKey === "line_ids")
-            return "line_ids";
-        if (lowerKey === "startpagekey" ||
-            lowerKey === "start_pagekey" ||
-            lowerKey === "start_page_key")
-            return "start_page_key";
-        if (lowerKey === "fileid" ||
-            lowerKey === "file_id" ||
-            lowerKey === "attachmentid" ||
-            lowerKey === "attachment_id")
-            return "attachment_id";
-        if (lowerKey === "keyspan" || lowerKey === "key_span")
-            return "key_span";
-        if (lowerKey === "reasoning" || lowerKey === "value")
-            return lowerKey;
-        if (lowerKey === "timestamps" ||
-            lowerKey === "timestamp" ||
-            lowerKey === "timestamps")
-            return "timestamps";
-        return lowerKey;
-    };
-    // Helper to decode HTML entities (simple implementation, expand if needed)
-    const decodeHtmlEntities = (str) => {
-        return str
-            .replace(/&quot;/g, '"')
-            .replace(/&apos;/g, "'")
-            .replace(/&lt;/g, "<")
-            .replace(/&gt;/g, ">")
-            .replace(/&amp;/g, "&");
-    };
-    // 2. ROBUST TEXT ATTRIBUTE PARSING (reasoning, value, full_phrase)
-    // This regex matches: Key = Quote -> Content (lazy) -> Lookahead for (Next Attribute OR End of Tag)
-    // It effectively ignores quotes inside the content during the initial capture.
-    const textAttributeRegex = /(fullPhrase|full_phrase|keySpan|key_span|reasoning|value)\s*=\s*(['"])([\s\S]*?)(?=\s+(?:line_ids|lineIds|timestamps|fileId|file_id|attachmentId|attachment_id|start_page_key|start_pageKey|startPageKey|keySpan|key_span|reasoning|value|full_phrase)|\s*\/?>)/gm;
-    normalized = normalized.replace(textAttributeRegex, (_match, key, openQuote, rawContent) => {
-        let content = rawContent;
-        // The lazy match usually captures the closing quote because the lookahead
-        // starts at the space *after* the attribute. We must strip it.
-        if (content.endsWith(openQuote)) {
-            content = content.slice(0, -1);
-        }
-        // 1. Normalization: Flatten newlines to spaces
-        content = content.replace(/(\r?\n)+/g, " ");
-        // 2. Decode entities to get raw text (e.g., &apos; -> ')
-        content = decodeHtmlEntities(content);
-        // 3. Remove Markdown bold/italic markers often hallucinated by LLMs inside attributes
-        content = content.replace(/(\*|_){2,}/g, "");
-        // 4. Sanitize Quotes:
-        // First, unescape existing backslashed quotes to avoid double escaping (e.g. \\' -> ')
-        content = content.replace(/\\\\'/g, "'");
-        content = content.replace(/\\'/g, "'");
-        content = content.replace(/'/g, "\\'");
-        content = content.replace(/\\\\"/g, '"');
-        content = content.replace(/\\"/g, '"');
-        content = content.replace(/"/g, '\\"');
-        // 5. Remove * from the content, sometimes a md list will really mess things up here so we remove it
-        content = content.replace(/\*/g, ""); //this is a hack to remove the * from the content
-        return `${canonicalizeCiteAttributeKey(key)}='${content}'`;
-    });
-    // 3. ROBUST LINE_ID / TIMESTAMP PARSING
-    // Handles unquoted, single quoted, or double quoted numbers/ranges.
-    // Can handle line_ids appearing anywhere in the tag, not just at the end.
-    normalized = normalized.replace(/(line_ids|lineIds|timestamps)=['"]?([\[\]\(\){}A-Za-z0-9_\-, ]+)['"]?(\s*\/?>|\s+)/gm, (_match, key, rawValue, trailingChars) => {
-        // Clean up the value (remove generic text, keep numbers/separators)
-        let cleanedValue = rawValue.replace(/[A-Za-z\[\]\(\){}]/g, "");
-        // Expand ranges (e.g., "1-3" -> "1,2,3")
-        cleanedValue = cleanedValue.replace(/(\d+)-(\d+)/g, (_rangeMatch, start, end) => {
-            const startNum = parseInt(start, 10);
-            const endNum = parseInt(end, 10);
-            const range = [];
-            // Handle ascending range
-            if (startNum <= endNum) {
-                for (let i = startNum; i <= endNum; i++) {
-                    range.push(i);
-                }
-            }
-            else {
-                // Fallback for weird descending ranges or just return start
-                range.push(startNum);
-            }
-            return range.join(",");
-        });
-        // Normalize commas
-        cleanedValue = cleanedValue.replace(/,+/g, ",").replace(/^,|,$/g, "");
-        // Return standardized format: key='value' + preserved trailing characters (space or />)
-        return `${canonicalizeCiteAttributeKey(key)}='${cleanedValue}'${trailingChars}`;
-    });
-    // 4. Re-order <cite ... /> attributes to match the strict parsing expectations in `citationParser.ts`
-    // (the parser uses regexes that assume a canonical attribute order).
-    const reorderCiteTagAttributes = (tag) => {
-        // Match both single-quoted and double-quoted attributes
-        const attrRegex = /([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(['"])((?:[^'"\\\n]|\\.)*)(?:\2)/g;
-        const attrs = {};
-        let match;
-        while ((match = attrRegex.exec(tag))) {
-            const rawKey = match[1];
-            const value = match[3]; // match[2] is the quote character
-            const key = canonicalizeCiteAttributeKey(rawKey);
-            attrs[key] = value;
-        }
-        // If we didn't find any parsable attrs, don't touch the tag.
-        const keys = Object.keys(attrs);
-        if (keys.length === 0)
-            return tag;
-        const hasTimestamps = typeof attrs.timestamps === "string" && attrs.timestamps.length > 0;
-        const startPageKeys = keys.filter((k) => k.startsWith("start_page"));
-        const ordered = [];
-        // Shared first
-        if (attrs.attachment_id)
-            ordered.push("attachment_id");
-        if (hasTimestamps) {
-            // AV citations: attachment_id, full_phrase, timestamps, (optional reasoning/value), then any extras
-            if (attrs.full_phrase)
-                ordered.push("full_phrase");
-            ordered.push("timestamps");
-        }
-        else {
-            // Document citations: attachment_id, start_page*, full_phrase, key_span, line_ids, (optional reasoning/value), then any extras
-            if (startPageKeys.includes("start_page_key"))
-                ordered.push("start_page_key");
-            startPageKeys
-                .filter((k) => k !== "start_page_key")
-                .sort()
-                .forEach((k) => ordered.push(k));
-            if (attrs.full_phrase)
-                ordered.push("full_phrase");
-            if (attrs.key_span)
-                ordered.push("key_span");
-            if (attrs.line_ids)
-                ordered.push("line_ids");
-        }
-        // Optional attrs supported by the parser (but not required)
-        if (attrs.reasoning)
-            ordered.push("reasoning");
-        if (attrs.value)
-            ordered.push("value");
-        // Any remaining attributes, stable + deterministic (alpha)
-        const used = new Set(ordered);
-        keys
-            .filter((k) => !used.has(k))
-            .sort()
-            .forEach((k) => ordered.push(k));
-        const rebuiltAttrs = ordered.map((k) => `${k}='${attrs[k]}'`).join(" ");
-        return `<cite ${rebuiltAttrs} />`;
-    };
-    normalized = normalized.replace(/<cite\b[\s\S]*?\/>/gm, (tag) => reorderCiteTagAttributes(tag));
-    return normalized;
-};

package/lib/parsing/parseCitation.d.ts

@@ -1,79 +0,0 @@
-import { type Verification } from "../types/verification.js";
-import { type Citation, type CitationStatus } from "../types/citation.js";
-/**
- * Calculates the verification status of a citation based on the found highlight and search state.
- *
- * @param verification - The found highlight location, or null/undefined if not found
- * @returns An object containing boolean flags for verification status
- */
-export declare function getCitationStatus(verification: Verification | null | undefined): CitationStatus;
-export declare const parseCitation: (fragment: string, mdAttachmentId?: string | null, citationCounterRef?: any | null, isVerbose?: boolean) => {
-    beforeCite: string;
-    afterCite: string;
-    citation: Citation;
-};
-/**
- * Extracts all citations from LLM output.
- * Supports both XML <cite ... /> tags (embedded in strings/markdown) and JSON-based citation formats.
- *
- * For object input:
- * - Traverses the object looking for `citation` or `citations` properties matching JSON format
- * - Also stringifies the object to find embedded XML citations in markdown content
- *
- * @param llmOutput - The LLM output (string or object)
- * @returns Dictionary of parsed Citation objects keyed by citation key
- */
-export declare const getAllCitationsFromLlmOutput: (llmOutput: any) => {
-    [key: string]: Citation;
-};
-/**
- * Groups citations by their attachmentId for multi-file verification scenarios.
- * This is useful when you have citations from multiple files and need to
- * verify them against their respective source documents.
- *
- * @param citations - Array of Citation objects or a dictionary of citations
- * @returns Map of attachmentId to dictionary of citations from that file
- *
- * @example
- * ```typescript
- * const citations = getAllCitationsFromLlmOutput(response.content);
- * const citationsByAttachment = groupCitationsByAttachmentId(citations);
- *
- * // Verify citations for each file
- * for (const [attachmentId, fileCitations] of citationsByAttachment) {
- *   const verified = await dc.verifyCitations(attachmentId, fileCitations);
- *   // Process verification results...
- * }
- * ```
- */
-export declare function groupCitationsByAttachmentId(citations: Citation[] | {
-    [key: string]: Citation;
-}): Map<string, {
-    [key: string]: Citation;
-}>;
-/**
- * Groups citations by their attachmentId and returns as a plain object.
- * Alternative to groupCitationsByAttachmentId that returns a plain object instead of a Map.
- *
- * @param citations - Array of Citation objects or a dictionary of citations
- * @returns Object with attachmentId keys mapping to citation dictionaries
- *
- * @example
- * ```typescript
- * const citations = getAllCitationsFromLlmOutput(response.content);
- * const citationsByAttachment = groupCitationsByAttachmentIdObject(citations);
- *
- * // Verify citations for each file using Promise.all
- * const verificationPromises = Object.entries(citationsByAttachment).map(
- *   ([attachmentId, fileCitations]) => dc.verifyCitations(attachmentId, fileCitations)
- * );
- * const results = await Promise.all(verificationPromises);
- * ```
- */
-export declare function groupCitationsByAttachmentIdObject(citations: Citation[] | {
-    [key: string]: Citation;
-}): {
-    [attachmentId: string]: {
-        [key: string]: Citation;
-    };
-};