@deepcitation/deepcitation-js 1.0.2 → 1.0.3

package/lib/parsing/parseCitation.js

@@ -0,0 +1,371 @@
+import { sha1Hash } from "../utils/sha.js";
+import { normalizeCitations } from "./normalizeCitation.js";
+import { generateCitationKey } from "../react/utils.js";
+/**
+ * Calculates the verification status of a citation based on the found highlight and search state.
+ *
+ * @param foundHighlight - The found highlight location, or null/undefined if not found
+ * @returns An object containing boolean flags for verification status
+ */
+export function getCitationStatus(foundHighlight) {
+    const searchState = foundHighlight?.searchState;
+    const isMiss = searchState?.status === "not_found";
+    const isFullMatchWithMissedValue = searchState?.status === "found_phrase_missed_value";
+    const isFoundValueMissedFullMatch = searchState?.status === "found_value_only";
+    const isPartialMatch = searchState?.status === "partial_text_found" ||
+        searchState?.status === "found_on_other_page" ||
+        searchState?.status === "found_on_other_line" ||
+        searchState?.status === "first_word_found";
+    const isVerified = searchState?.status === "found" || isFoundValueMissedFullMatch || isPartialMatch || isFullMatchWithMissedValue;
+    const isPending = searchState?.status === "pending" || searchState?.status === "loading" || !searchState;
+    return { isVerified, isMiss, isPartialMatch, isPending };
+}
+export const parseCitation = (fragment, mdAttachmentId, citationCounterRef, isVerbose) => {
+    // Helper: Remove wrapper quotes and unescape internal single quotes (e.g. It\'s -> It's)
+    const cleanAndUnescape = (str) => {
+        if (!str)
+            return undefined;
+        // Remove surrounding quotes if present (regex usually handles this, but safety first)
+        const trimmed = str.replace(/^['"]|['"]$/g, "");
+        // Replace escaped single quotes with actual single quotes
+        return trimmed.replace(/\\'/g, "'");
+    };
+    const citationNumber = citationCounterRef?.current ? citationCounterRef.current++ : undefined;
+    const beforeCite = fragment.substring(0, fragment.indexOf("<cite"));
+    const afterCite = fragment.includes("/>") ? fragment.slice(fragment.indexOf("/>") + 2) : "";
+    const middleCite = fragment.substring(fragment.indexOf("<cite"), fragment.indexOf("/>") + 2);
+    // GROUPS:
+    // 1: fileId
+    // 2: start_page number
+    // 3: index number
+    // 4: full_phrase content (escaped)
+    // 5: line_ids content
+    // 6: Optional Key (value|reasoning)
+    // 7: Optional Value content (escaped)
+    const citationRegex = /<cite\s+file(?:_id|Id)='(\w{0,25})'\s+start_page[\_a-zA-Z]*='page[\_a-zA-Z]*(\d+)_index_(\d+)'\s+full_phrase='((?:[^'\\]|\\.)*)'\s+line(?:_ids|Ids)='([^']+)'(?:\s+(value|reasoning)='((?:[^'\\]|\\.)*)')?\s*\/>/g;
+    const citationMatches = [...middleCite.matchAll(citationRegex)];
+    const match = citationMatches?.[0];
+    const rawCitationMd = match?.[0];
+    const pageNumber = match?.[2] ? parseInt(match?.[2]) : undefined;
+    let fileId = match?.[1];
+    let attachmentId = fileId?.length === 20 ? fileId : mdAttachmentId || match?.[1];
+    // Use helper to handle escaped quotes inside the phrase
+    let fullPhrase = cleanAndUnescape(match?.[4]);
+    // Handle the optional attribute (value or reasoning)
+    let value;
+    let reasoning;
+    const optionalKey = match?.[6]; // "value" or "reasoning"
+    const optionalContent = cleanAndUnescape(match?.[7]);
+    if (optionalKey === "value") {
+        value = optionalContent;
+    }
+    else if (optionalKey === "reasoning") {
+        reasoning = optionalContent;
+    }
+    let lineIds;
+    try {
+        // match[5] is line_ids
+        const lineIdsString = match?.[5]?.replace(/[A-Za-z_[\](){}:]/g, "");
+        lineIds = lineIdsString
+            ? lineIdsString
+                .split(",")
+                .map(id => (isNaN(parseInt(id)) ? undefined : parseInt(id)))
+                .filter(id => id !== undefined)
+                .sort((a, b) => a - b)
+            : undefined;
+    }
+    catch (e) {
+        if (isVerbose)
+            console.error("Error parsing lineIds", e);
+    }
+    // GROUPS for AV:
+    // 1: fileId
+    // 2: full_phrase content (escaped)
+    // 3: timestamps content
+    // 4: Optional Key (value|reasoning)
+    // 5: Optional Value content (escaped)
+    const avCitationRegex = /<cite\s+file(?:_id|Id)='(\w{0,25})'\s+full_phrase='((?:[^'\\]|\\.)*)'\s+timestamps='([^']+)'(?:\s+(value|reasoning)='((?:[^'\\]|\\.)*)')?\s*\/>/g;
+    const avCitationMatches = [...middleCite.matchAll(avCitationRegex)];
+    const avMatch = avCitationMatches?.[0];
+    let timestamps;
+    if (avMatch) {
+        fileId = avMatch?.[1];
+        attachmentId = fileId?.length === 20 ? fileId : mdAttachmentId || avMatch?.[1];
+        fullPhrase = cleanAndUnescape(avMatch?.[2]);
+        const timestampsString = avMatch?.[3]?.replace(/timestamps=['"]|['"]/g, "");
+        const [startTime, endTime] = timestampsString?.split("-") || [];
+        const avOptionalKey = avMatch?.[4];
+        const avOptionalContent = cleanAndUnescape(avMatch?.[5]);
+        if (avOptionalKey === "value") {
+            value = avOptionalContent;
+        }
+        else if (avOptionalKey === "reasoning") {
+            reasoning = avOptionalContent;
+        }
+        timestamps = { startTime, endTime };
+    }
+    const fragmentContext = sha1Hash(fragment).toString().slice(0, 8);
+    const citation = {
+        fragmentContext,
+        fileId: attachmentId,
+        pageNumber,
+        fullPhrase,
+        citationNumber,
+        lineIds,
+        rawCitationMd,
+        beforeCite,
+        value,
+        timestamps,
+        reasoning,
+    };
+    return {
+        beforeCite,
+        afterCite,
+        citation,
+    };
+};
+/**
+ * Parses a JSON-based citation object into a Citation.
+ * Supports both camelCase and snake_case property names.
+ *
+ * @param jsonCitation - The JSON citation object (can have camelCase or snake_case properties)
+ * @param citationNumber - Optional citation number for ordering
+ * @returns Parsed Citation object
+ */
+const parseJsonCitation = (jsonCitation, citationNumber) => {
+    if (!jsonCitation) {
+        return null;
+    }
+    // Support both camelCase and snake_case property names
+    const fullPhrase = jsonCitation.fullPhrase ?? jsonCitation.full_phrase;
+    const startPageKey = jsonCitation.startPageKey ?? jsonCitation.start_page_key;
+    const rawLineIds = jsonCitation.lineIds ?? jsonCitation.line_ids;
+    const fileId = jsonCitation.fileId ?? jsonCitation.file_id;
+    const reasoning = jsonCitation.reasoning;
+    const value = jsonCitation.value;
+    if (!fullPhrase) {
+        return null;
+    }
+    // Parse startPageKey format: "page_number_PAGE_index_INDEX"
+    let pageNumber;
+    if (startPageKey) {
+        const pageMatch = startPageKey.match(/page[_a-zA-Z]*(\d+)_index_(\d+)/i);
+        if (pageMatch) {
+            pageNumber = parseInt(pageMatch[1], 10);
+        }
+    }
+    // Sort lineIds if present
+    const lineIds = rawLineIds?.length ? [...rawLineIds].sort((a, b) => a - b) : undefined;
+    const citation = {
+        fileId,
+        pageNumber,
+        fullPhrase,
+        citationNumber,
+        lineIds,
+        reasoning,
+        value,
+    };
+    return citation;
+};
+/**
+ * Checks if an object has citation-like properties (camelCase or snake_case).
+ */
+const hasCitationProperties = (item) => typeof item === "object" &&
+    item !== null &&
+    ("fullPhrase" in item ||
+        "full_phrase" in item ||
+        "startPageKey" in item ||
+        "start_page_key" in item ||
+        "lineIds" in item ||
+        "line_ids" in item);
+/**
+ * Checks if the input appears to be JSON-based citations.
+ * Looks for array of objects with citation-like properties (supports both camelCase and snake_case).
+ */
+const isJsonCitationFormat = (data) => {
+    if (Array.isArray(data)) {
+        return data.length > 0 && data.some(hasCitationProperties);
+    }
+    if (typeof data === "object" && data !== null) {
+        return hasCitationProperties(data);
+    }
+    return false;
+};
+/**
+ * Extracts citations from JSON format (array or single object).
+ */
+const extractJsonCitations = (data) => {
+    const citations = {};
+    const items = Array.isArray(data) ? data : [data];
+    let citationNumber = 1;
+    for (const item of items) {
+        const citation = parseJsonCitation(item, citationNumber++);
+        if (citation && citation.fullPhrase) {
+            const citationKey = generateCitationKey(citation);
+            citations[citationKey] = citation;
+        }
+    }
+    return citations;
+};
+/**
+ * Recursively traverses an object looking for `citation` or `citations` properties
+ * that match our JSON citation format.
+ */
+const findJsonCitationsInObject = (obj, found) => {
+    if (!obj || typeof obj !== "object")
+        return;
+    // Check for citation/citations properties
+    if (obj.citation && isJsonCitationFormat(obj.citation)) {
+        const items = Array.isArray(obj.citation) ? obj.citation : [obj.citation];
+        found.push(...items);
+    }
+    if (obj.citations && isJsonCitationFormat(obj.citations)) {
+        const items = Array.isArray(obj.citations) ? obj.citations : [obj.citations];
+        found.push(...items);
+    }
+    // Recurse into object properties
+    if (Array.isArray(obj)) {
+        for (const item of obj) {
+            findJsonCitationsInObject(item, found);
+        }
+    }
+    else {
+        for (const key of Object.keys(obj)) {
+            if (key !== "citation" && key !== "citations") {
+                findJsonCitationsInObject(obj[key], found);
+            }
+        }
+    }
+};
+/**
+ * Extracts XML citations from text using <cite ... /> tags.
+ */
+const extractXmlCitations = (text) => {
+    const normalizedText = normalizeCitations(text);
+    // Find all <cite ... /> tags
+    const citeRegex = /<cite\s+[^>]*\/>/g;
+    const matches = normalizedText.match(citeRegex);
+    if (!matches || matches.length === 0)
+        return {};
+    const citations = {};
+    const citationCounterRef = { current: 1 };
+    for (const match of matches) {
+        const { citation } = parseCitation(match, undefined, citationCounterRef);
+        if (citation && citation.fullPhrase) {
+            const citationKey = generateCitationKey(citation);
+            citations[citationKey] = citation;
+        }
+    }
+    return citations;
+};
+/**
+ * Extracts all citations from LLM output.
+ * Supports both XML <cite ... /> tags (embedded in strings/markdown) and JSON-based citation formats.
+ *
+ * For object input:
+ * - Traverses the object looking for `citation` or `citations` properties matching JSON format
+ * - Also stringifies the object to find embedded XML citations in markdown content
+ *
+ * @param llmOutput - The LLM output (string or object)
+ * @returns Dictionary of parsed Citation objects keyed by citation key
+ */
+export const getAllCitationsFromLlmOutput = (llmOutput) => {
+    if (!llmOutput)
+        return {};
+    const citations = {};
+    if (typeof llmOutput === "object") {
+        // Check if the root object itself is JSON citation format
+        if (isJsonCitationFormat(llmOutput)) {
+            const jsonCitations = extractJsonCitations(llmOutput);
+            Object.assign(citations, jsonCitations);
+        }
+        else {
+            // Traverse object for nested citation/citations properties
+            const foundJsonCitations = [];
+            findJsonCitationsInObject(llmOutput, foundJsonCitations);
+            if (foundJsonCitations.length > 0) {
+                const jsonCitations = extractJsonCitations(foundJsonCitations);
+                Object.assign(citations, jsonCitations);
+            }
+        }
+        // Also stringify and parse for embedded XML citations in markdown
+        const text = JSON.stringify(llmOutput);
+        const xmlCitations = extractXmlCitations(text);
+        Object.assign(citations, xmlCitations);
+    }
+    else if (typeof llmOutput === "string") {
+        // String input - parse for XML citations
+        const xmlCitations = extractXmlCitations(llmOutput);
+        Object.assign(citations, xmlCitations);
+    }
+    return citations;
+};
+/**
+ * Groups citations by their fileId for multi-file verification scenarios.
+ * This is useful when you have citations from multiple files and need to
+ * verify them against their respective source documents.
+ *
+ * @param citations - Array of Citation objects or a dictionary of citations
+ * @returns Map of fileId to dictionary of citations from that file
+ *
+ * @example
+ * ```typescript
+ * const citations = getAllCitationsFromLlmOutput(response.content);
+ * const citationsByFile = groupCitationsByFileId(citations);
+ *
+ * // Verify citations for each file
+ * for (const [fileId, fileCitations] of citationsByFile) {
+ *   const verified = await dc.verifyCitations(fileId, fileCitations);
+ *   // Process verification results...
+ * }
+ * ```
+ */
+export function groupCitationsByFileId(citations) {
+    const grouped = new Map();
+    // Normalize input to entries
+    const entries = Array.isArray(citations)
+        ? citations.map((c, idx) => [generateCitationKey(c) || String(idx + 1), c])
+        : Object.entries(citations);
+    for (const [key, citation] of entries) {
+        const fileId = citation.fileId || "";
+        if (!grouped.has(fileId)) {
+            grouped.set(fileId, {});
+        }
+        grouped.get(fileId)[key] = citation;
+    }
+    return grouped;
+}
+/**
+ * Groups citations by their fileId and returns as a plain object.
+ * Alternative to groupCitationsByFileId that returns a plain object instead of a Map.
+ *
+ * @param citations - Array of Citation objects or a dictionary of citations
+ * @returns Object with fileId keys mapping to citation dictionaries
+ *
+ * @example
+ * ```typescript
+ * const citations = getAllCitationsFromLlmOutput(response.content);
+ * const citationsByFile = groupCitationsByFileIdObject(citations);
+ *
+ * // Verify citations for each file using Promise.all
+ * const verificationPromises = Object.entries(citationsByFile).map(
+ *   ([fileId, fileCitations]) => dc.verifyCitations(fileId, fileCitations)
+ * );
+ * const results = await Promise.all(verificationPromises);
+ * ```
+ */
+export function groupCitationsByFileIdObject(citations) {
+    const grouped = {};
+    // Normalize input to entries
+    const entries = Array.isArray(citations)
+        ? citations.map((c, idx) => [generateCitationKey(c) || String(idx + 1), c])
+        : Object.entries(citations);
+    for (const [key, citation] of entries) {
+        const fileId = citation.fileId || "";
+        if (!grouped[fileId]) {
+            grouped[fileId] = {};
+        }
+        grouped[fileId][key] = citation;
+    }
+    return grouped;
+}

package/lib/parsing/parseWorkAround.d.ts

@@ -0,0 +1,2 @@
+export declare const isGeminiGarbage: (content: string) => boolean;
+export declare function cleanRepeatingLastSentence(text: string): string;

package/lib/parsing/parseWorkAround.js

@@ -0,0 +1,73 @@
+//flash and flash lite get super confused if we ask for a MD table and infinite loop
+const MIN_CONTENT_LENGTH_FOR_GEMINI_GARBAGE = 64;
+export const isGeminiGarbage = (content) => {
+    if (!content)
+        return false;
+    const trimmedContent = content.trim();
+    if (trimmedContent.length < MIN_CONTENT_LENGTH_FOR_GEMINI_GARBAGE)
+        return false;
+    const firstCharacter = trimmedContent?.[0];
+    for (let i = 1; i < trimmedContent.length; i++) {
+        if (trimmedContent[i] !== firstCharacter)
+            return false;
+    }
+    return true;
+};
+// helps clean up infinite rambling bug output from gemini
+export function cleanRepeatingLastSentence(text) {
+    text = text.trim();
+    const MIN_REPETITIONS = 2;
+    const MIN_SENTENCE_CONTENT_LENGTH = 10;
+    const sentenceEndRegex = /[.?!](?=\s+|$)/g;
+    let match;
+    const sentenceEndIndices = [];
+    while ((match = sentenceEndRegex.exec(text)) !== null) {
+        sentenceEndIndices.push(match.index);
+    }
+    if (sentenceEndIndices.length < 2) {
+        return text;
+    }
+    const lastTerminatorIndex = sentenceEndIndices[sentenceEndIndices.length - 1];
+    const secondLastTerminatorIndex = sentenceEndIndices[sentenceEndIndices.length - 2];
+    const repeatingUnit = text.substring(secondLastTerminatorIndex + 1, lastTerminatorIndex + 1);
+    const unitLength = repeatingUnit.length;
+    const sentenceContent = repeatingUnit.trim().slice(0, -1);
+    if (sentenceContent.length < MIN_SENTENCE_CONTENT_LENGTH) {
+        return text;
+    }
+    if (unitLength <= 0) {
+        return text;
+    }
+    if (text.length < unitLength * MIN_REPETITIONS) {
+        return text;
+    }
+    let repetitionsFound = 0;
+    let currentCheckEndIndex = lastTerminatorIndex + 1;
+    if (text.endsWith(repeatingUnit)) {
+        currentCheckEndIndex = text.length;
+    }
+    let firstRepetitionStartIndex = -1;
+    while (true) {
+        const checkStartIndex = currentCheckEndIndex - unitLength;
+        if (checkStartIndex < 0) {
+            break;
+        }
+        const chunk = text.substring(checkStartIndex, currentCheckEndIndex);
+        if (chunk === repeatingUnit) {
+            repetitionsFound++;
+            firstRepetitionStartIndex = checkStartIndex;
+            currentCheckEndIndex = checkStartIndex;
+        }
+        else {
+            break;
+        }
+    }
+    if (repetitionsFound >= MIN_REPETITIONS) {
+        const textBeforeRepetitions = text.substring(0, firstRepetitionStartIndex);
+        const result = textBeforeRepetitions + repeatingUnit;
+        return result;
+    }
+    else {
+        return text;
+    }
+}

package/lib/prompts/citationPrompts.d.ts

@@ -0,0 +1,133 @@
+export declare const CITATION_MARKDOWN_SYNTAX_PROMPT = "\nCitation syntax to use within Markdown:\n\u2022 To support any ideas or information that requires a citation from the provided content, use the following citation syntax:\n<cite file_id='file_id' start_page_key='page_number_PAGE_index_INDEX' full_phrase='the verbatim text of the terse phrase inside <file_text /> (remember to escape quotes and newlines inside the full_phrase to remain as valid JSON)' line_ids='2-6' reasoning='the terse logic used to conclude the citation' />\n\n\u2022 Very important: for page numbers, only use the page number and page index info from the page_number_PAGE_index_INDEX format (e.g. <page_number_1_index_0>) and never from the contents inside the page.\n\u2022 start_page_key, full_phrase, and line_ids are required for each citation.\n\u2022 Infer line_ids, as we only provide the first, last, and every 5th line. When copying a previous <cite />, use the full info from the previous citation without changing the start_page_key, line_ids, or any other <cite /> attributes.\n\u2022 Use refer to line_ids inclusively, and use a range (or single) for each citation, split multiple sequential line_ids into multiple citations.\n\u2022 These citations will be replaced and displayed in-line as a numeric element (e.g. [1]), the markdown preceding <cite /> should read naturally with only one <cite /> per sentence with rare exceptions for two <cite /> in a sentence. <cite /> often present best at the end of the sentence, and are not grouped at the end of the document.\n\u2022 The full_phrase should be the exact verbatim text of the phrase or paragraph from the source document to support the insight or idea.\n\u2022 We do NOT put the full_phrase inside <cite ...></cite>; we only use full_phrase inside the full_phrase attribute.\n";
+export declare const AV_CITATION_MARKDOWN_SYNTAX_PROMPT = "\n\u2022 To support any ideas or information that requires a citation from the provided content, use the following citation syntax:\n<cite file_id='file_id' full_phrase='the verbatim text of the phrase (remember to escape quotes and newlines inside the full_phrase to remain as valid JSON)' timestamps='HH:MM:SS.SSS-HH:MM:SS.SSS' reasoning='the logic connecting the form section requirements to the supporting source citation' />\n\u2022 These citations are displayed in-line or in the relevant list item, and are not grouped at the end of the document.\n";
+export interface WrapSystemPromptOptions {
+    /** The original system prompt to wrap with citation instructions */
+    systemPrompt: string;
+    /** Whether to use audio/video citation format (with timestamps) instead of text-based (with line IDs) */
+    isAudioVideo?: boolean;
+    prependCitationInstructions?: boolean;
+}
+export interface WrapCitationPromptOptions {
+    /** The original system prompt to wrap with citation instructions */
+    systemPrompt: string;
+    /** The original user prompt */
+    userPrompt: string;
+    /** The extracted file text with metadata (from uploadFile response). Can be a single string or array for multiple files. */
+    fileDeepText?: string | string[];
+    /** Whether to use audio/video citation format (with timestamps) instead of text-based (with line IDs) */
+    isAudioVideo?: boolean;
+}
+export interface WrapCitationPromptResult {
+    /** Enhanced system prompt with citation instructions */
+    enhancedSystemPrompt: string;
+    /** Enhanced user prompt (currently passed through unchanged) */
+    enhancedUserPrompt: string;
+}
+/**
+ * Wraps your existing system prompt with DeepCitation's citation syntax instructions.
+ * This enables LLMs to output verifiable citations that can be checked against source documents.
+ *
+ * @example
+ * ```typescript
+ * import { wrapSystemCitationPrompt } from '@deepcitation/deepcitation-js';
+ *
+ * const systemPrompt = "You are a helpful assistant that analyzes documents.";
+ * const enhanced = wrapSystemCitationPrompt({ systemPrompt });
+ *
+ * // Use enhanced prompt with your LLM
+ * const response = await openai.chat.completions.create({
+ *   messages: [{ role: "system", content: enhanced }],
+ *   // ...
+ * });
+ * ```
+ */
+export declare function wrapSystemCitationPrompt(options: WrapSystemPromptOptions): string;
+/**
+ * Wraps both system and user prompts with DeepCitation's citation syntax instructions.
+ * This is the recommended way to prepare prompts for citation verification.
+ *
+ * @example
+ * ```typescript
+ * import { wrapCitationPrompt } from '@deepcitation/deepcitation-js';
+ *
+ * // Single file
+ * const { enhancedSystemPrompt, enhancedUserPrompt } = wrapCitationPrompt({
+ *   systemPrompt: "You are a helpful assistant.",
+ *   userPrompt: "Analyze this document and summarize it.",
+ *   fileDeepText, // from uploadFile response
+ * });
+ *
+ * // Multiple files
+ * const { enhancedSystemPrompt, enhancedUserPrompt } = wrapCitationPrompt({
+ *   systemPrompt: "You are a helpful assistant.",
+ *   userPrompt: "Compare these documents.",
+ *   fileDeepText: [fileDeepText1, fileDeepText2], // array of file texts
+ * });
+ *
+ * // Use enhanced prompts with your LLM
+ * const response = await llm.chat({
+ *   messages: [
+ *     { role: "system", content: enhancedSystemPrompt },
+ *     { role: "user", content: enhancedUserPrompt },
+ *   ],
+ * });
+ * ```
+ */
+export declare function wrapCitationPrompt(options: WrapCitationPromptOptions): WrapCitationPromptResult;
+export declare const CITATION_JSON_OUTPUT_FORMAT: {
+    type: string;
+    properties: {
+        fileId: {
+            type: string;
+        };
+        startPageKey: {
+            type: string;
+            description: string;
+        };
+        reasoning: {
+            type: string;
+            description: string;
+        };
+        fullPhrase: {
+            type: string;
+            description: string;
+        };
+        lineIds: {
+            type: string;
+            items: {
+                type: string;
+            };
+            description: string;
+        };
+    };
+    required: string[];
+};
+export declare const CITATION_AV_BASED_JSON_OUTPUT_FORMAT: {
+    type: string;
+    properties: {
+        fileId: {
+            type: string;
+        };
+        startPageKey: {
+            type: string;
+            description: string;
+        };
+        fullPhrase: {
+            type: string;
+            description: string;
+        };
+        timestamps: {
+            type: string;
+            properties: {
+                startTime: {
+                    type: string;
+                };
+                endTime: {
+                    type: string;
+                };
+            };
+            required: string[];
+            description: string;
+        };
+    };
+};