@deepcitation/deepcitation-js 1.0.2 → 1.0.4

package/lib/client/types.d.ts

@@ -0,0 +1,157 @@
+import type { Citation, FoundHighlightLocation } from "../types/index";
+/**
+ * Configuration options for the DeepCitation client
+ */
+export interface DeepCitationConfig {
+    /** Your DeepCitation API key (starts with dc_live_ or dc_test_) */
+    apiKey: string;
+    /** Optional custom API base URL. Defaults to https://api.deepcitation.com */
+    apiUrl?: string;
+}
+/**
+ * Response from uploading a file for citation verification
+ */
+export interface UploadFileResponse {
+    /** The file ID assigned by DeepCitation (custom or auto-generated) */
+    fileId: string;
+    /** The full text content formatted for LLM prompts with page markers and line IDs. Use this in your user prompts. */
+    fileDeepText: string;
+    /** Form fields extracted from PDF forms */
+    formFields?: Array<{
+        name: string;
+        value?: string;
+        pageIndex?: number;
+        type?: string;
+    }>;
+    /** Metadata about the processed file */
+    metadata: {
+        filename: string;
+        mimeType: string;
+        pageCount: number;
+        textByteSize: number;
+    };
+    /** Processing status */
+    status: "ready" | "error";
+    /** Time taken to process the file in milliseconds */
+    processingTimeMs?: number;
+    /** Error message if status is "error" */
+    error?: string;
+}
+/**
+ * Options for file upload
+ */
+export interface UploadFileOptions {
+    /** Optional custom file ID to use instead of auto-generated one */
+    fileId?: string;
+    /** Optional custom filename (uses File.name if not provided) */
+    filename?: string;
+}
+/**
+ * Response from verifying citations
+ */
+export interface VerifyCitationsResponse {
+    /** Map of citation keys to their verification results */
+    foundHighlights: Record<string, FoundHighlightLocation>;
+}
+/**
+ * Options for citation verification
+ */
+export interface VerifyCitationsOptions {
+    /** Output image format for verification screenshots */
+    outputImageFormat?: "jpeg" | "png" | "avif";
+}
+/**
+ * Simplified citation input for verification
+ */
+export type CitationInput = Citation | Record<string, Citation>;
+/**
+ * Input for file upload in prepareFiles
+ */
+export interface FileInput {
+    /** The file content (File, Blob, or Buffer) */
+    file: File | Blob | Buffer;
+    /** Optional filename */
+    filename?: string;
+    /** Optional custom file ID */
+    fileId?: string;
+}
+/**
+ * File reference returned from prepareFiles
+ */
+export interface FileDataPart {
+    /** The file ID assigned by DeepCitation */
+    fileId: string;
+}
+/**
+ * Result from prepareFiles
+ */
+export interface PrepareFilesResult {
+    /** Array of file references for verification */
+    fileDataParts: FileDataPart[];
+    /** Array of formatted text content for LLM prompts (with page markers and line IDs) */
+    fileDeepTexts: string[];
+}
+/**
+ * Input for verifyCitationsFromLlmOutput
+ */
+export interface VerifyCitationsFromLlmOutputInput {
+    /** The LLM response containing citations */
+    llmOutput: string;
+    /** Optional file references (required for Zero Data Retention or after storage expires) */
+    fileDataParts?: FileDataPart[];
+    /** Output image format for verification screenshots */
+    outputImageFormat?: "jpeg" | "png" | "avif";
+}
+/**
+ * Input for convertFile - convert URL or Office file to PDF
+ */
+export interface ConvertFileInput {
+    /** URL to convert to PDF (for web pages or direct PDF links) */
+    url?: string;
+    /** Office file to convert (doc, docx, xls, xlsx, ppt, pptx, odt, ods, odp) */
+    file?: File | Blob | Buffer;
+    /** Optional custom filename for the converted PDF */
+    filename?: string;
+    /** Optional custom file ID */
+    fileId?: string;
+    /** For URLs: render as single long page instead of paginated */
+    singlePage?: boolean;
+}
+/**
+ * Response from convertFile
+ */
+export interface ConvertFileResponse {
+    /** The file ID assigned by DeepCitation. Pass this to prepareConvertedFile(). */
+    fileId: string;
+    /** Metadata about the conversion */
+    metadata: {
+        /** Original filename before conversion */
+        originalFilename: string;
+        /** Original MIME type before conversion */
+        originalMimeType: string;
+        /** MIME type after conversion (always application/pdf) */
+        convertedMimeType: string;
+        /** Time taken for conversion in milliseconds */
+        conversionTimeMs: number;
+    };
+    /** Conversion status */
+    status: "converted" | "error";
+    /** Error message if status is "error" */
+    error?: string;
+}
+/**
+ * Options for processing a converted file
+ */
+export interface PrepareConvertedFileOptions {
+    /** The file ID from a previous convertFile call */
+    fileId: string;
+}
+/**
+ * @deprecated Use PrepareConvertedFileOptions instead
+ */
+export interface PrepareFileFromAttachmentOptions {
+    /** The attachment ID from a previous convertFile call */
+    attachmentId: string;
+    /** Optional custom file ID */
+    fileId?: string;
+}

package/lib/client/types.js

@@ -0,0 +1 @@
+export {};

package/lib/index.d.ts

@@ -0,0 +1,25 @@
+/**
+ * DeepCitation - Citation parsing, verification, and rendering library
+ * @packageDocumentation
+ */
+export { DeepCitation } from "./client/index.js";
+export type { DeepCitationConfig, UploadFileResponse, UploadFileOptions, VerifyCitationsResponse, VerifyCitationsOptions, CitationInput, FileInput, FileDataPart, PrepareFilesResult, VerifyCitationsFromLlmOutputInput, } from "./client/index.js";
+export { parseCitation, getCitationStatus, getAllCitationsFromLlmOutput, groupCitationsByFileId, groupCitationsByFileIdObject, } from "./parsing/parseCitation.js";
+export { normalizeCitations, getCitationPageNumber } from "./parsing/normalizeCitation.js";
+export { isGeminiGarbage, cleanRepeatingLastSentence } from "./parsing/parseWorkAround.js";
+export type { Citation, CitationStatus, VerifyCitationRequest, VerifyCitationResponse, OutputImageFormat, } from "./types/citation.js";
+export { VERIFICATION_VERSION_NUMBER, DEFAULT_OUTPUT_IMAGE_FORMAT } from "./types/citation.js";
+export type { FoundHighlightLocation } from "./types/foundHighlight.js";
+export { NOT_FOUND_HIGHLIGHT_INDEX, PENDING_HIGHLIGHT_INDEX, BLANK_HIGHLIGHT_LOCATION, deterministicIdFromHighlightLocation, } from "./types/foundHighlight.js";
+export type { SearchState, SearchStatus, SearchMethod, SearchAttempt } from "./types/search.js";
+export type { ScreenBox, PdfSpaceItem, IVertex } from "./types/boxes.js";
+export { sha1Hash } from "./utils/sha.js";
+export { generateCitationKey } from "./react/utils.js";
+export { generateCitationInstanceId } from "./react/utils.js";
+export { CITATION_X_PADDING, CITATION_Y_PADDING } from "./react/utils.js";
+export { CITATION_JSON_OUTPUT_FORMAT, CITATION_MARKDOWN_SYNTAX_PROMPT, AV_CITATION_MARKDOWN_SYNTAX_PROMPT, CITATION_AV_BASED_JSON_OUTPUT_FORMAT, wrapSystemCitationPrompt, wrapCitationPrompt, } from "./prompts/citationPrompts.js";
+export type { WrapSystemPromptOptions, WrapCitationPromptOptions, WrapCitationPromptResult, } from "./prompts/citationPrompts.js";
+export { removeLineIdMetadata, removePageNumberMetadata, removeCitations } from "./parsing/normalizeCitation.js";
+export { compressPromptIds, decompressPromptIds } from "./prompts/promptCompression.js";
+export type { CompressedResult } from "./prompts/types.js";
+export { CitationComponent } from "./react/CitationComponent.js";

package/lib/index.js

@@ -0,0 +1,22 @@
+/**
+ * DeepCitation - Citation parsing, verification, and rendering library
+ * @packageDocumentation
+ */
+// Client
+export { DeepCitation } from "./client/index.js";
+// Parsing
+export { parseCitation, getCitationStatus, getAllCitationsFromLlmOutput, groupCitationsByFileId, groupCitationsByFileIdObject, } from "./parsing/parseCitation.js";
+export { normalizeCitations, getCitationPageNumber } from "./parsing/normalizeCitation.js";
+export { isGeminiGarbage, cleanRepeatingLastSentence } from "./parsing/parseWorkAround.js";
+export { VERIFICATION_VERSION_NUMBER, DEFAULT_OUTPUT_IMAGE_FORMAT } from "./types/citation.js";
+export { NOT_FOUND_HIGHLIGHT_INDEX, PENDING_HIGHLIGHT_INDEX, BLANK_HIGHLIGHT_LOCATION, deterministicIdFromHighlightLocation, } from "./types/foundHighlight.js";
+// Utilities
+export { sha1Hash } from "./utils/sha.js";
+export { generateCitationKey } from "./react/utils.js";
+export { generateCitationInstanceId } from "./react/utils.js";
+export { CITATION_X_PADDING, CITATION_Y_PADDING } from "./react/utils.js";
+// Prompts
+export { CITATION_JSON_OUTPUT_FORMAT, CITATION_MARKDOWN_SYNTAX_PROMPT, AV_CITATION_MARKDOWN_SYNTAX_PROMPT, CITATION_AV_BASED_JSON_OUTPUT_FORMAT, wrapSystemCitationPrompt, wrapCitationPrompt, } from "./prompts/citationPrompts.js";
+export { removeLineIdMetadata, removePageNumberMetadata, removeCitations } from "./parsing/normalizeCitation.js";
+export { compressPromptIds, decompressPromptIds } from "./prompts/promptCompression.js";
+export { CitationComponent } from "./react/CitationComponent.js";

package/lib/parsing/normalizeCitation.d.ts

@@ -0,0 +1,5 @@
+export declare const removeCitations: (pageText: string, leaveValueBehind?: boolean) => string;
+export declare const removePageNumberMetadata: (pageText: string) => string;
+export declare const removeLineIdMetadata: (pageText: string) => string;
+export declare const getCitationPageNumber: (startPageKey?: string | null) => number | null;
+export declare const normalizeCitations: (response: string) => string;

package/lib/parsing/normalizeCitation.js

@@ -0,0 +1,182 @@
+export const removeCitations = (pageText, leaveValueBehind) => {
+    const citationRegex = /<cite\s+fileId='(\w{0,25})'\s+start_page[\_a-zA-Z]*='page[\_a-zA-Z]*(\d+)_index_(\d+)'\s+full_phrase='((?:[^'\\]|\\.)*)'\s+line(?:_ids|Ids)='([^']+)'(?:\s+(value|reasoning)='((?:[^'\\]|\\.)*)')?\s*\/>/g;
+    return pageText.replace(citationRegex, (match, fileId, pageNumber, index, fullPhrase, lineIds, value) => {
+        //it is still value= so we need to remove the value=
+        if (leaveValueBehind) {
+            return value?.replace(/value=['"]|['"]/g, "") || "";
+        }
+        else {
+            return "";
+        }
+    });
+};
+export const removePageNumberMetadata = (pageText) => {
+    return pageText
+        .replace(/<page_number_\d+_index_\d+>/g, "")
+        .replace(/<\/page_number_\d+_index_\d+>/g, "")
+        .trim();
+};
+export const removeLineIdMetadata = (pageText) => {
+    const lineIdRegex = /<line id="[^"]*">|<\/line>/g;
+    return pageText.replace(lineIdRegex, "");
+};
+export const getCitationPageNumber = (startPageKey) => {
+    //page_number_{page_number}_index_{page_index} or page_number_{page_number} or page_key_{page_number}_index_{page_index}
+    if (!startPageKey)
+        return null;
+    //regex first \d+ is the page number
+    const pageNumber = startPageKey.match(/\d+/)?.[0];
+    return pageNumber ? parseInt(pageNumber) : null;
+};
+export const normalizeCitations = (response) => {
+    let trimmedResponse = response?.trim() || "";
+    const citationParts = trimmedResponse.split(/(<cite[\s\S]*?(?:\/>|<\/cite>))/gm);
+    if (citationParts.length <= 1) {
+        return normalizeCitationContent(trimmedResponse);
+    }
+    trimmedResponse = citationParts
+        .map(part => (part.startsWith("<cite") ? normalizeCitationContent(part) : part))
+        .join("");
+    return trimmedResponse;
+};
+const normalizeCitationContent = (input) => {
+    let normalized = input;
+    // 1. Standardize self-closing tags
+    // Replace ></cite> with /> for consistency
+    normalized = normalized.replace(/><\/cite>/g, "/>");
+    const canonicalizeCiteAttributeKey = (key) => {
+        if (key === "fullPhrase" || key === "full_phrase")
+            return "full_phrase";
+        if (key === "lineIds" || key === "line_ids")
+            return "line_ids";
+        if (key === "startPageKey" || key === "start_pageKey" || key === "start_page_key")
+            return "start_page_key";
+        if (key === "fileID" || key === "fileId" || key === "file_id")
+            return "file_id";
+        return key;
+    };
+    // Helper to decode HTML entities (simple implementation, expand if needed)
+    const decodeHtmlEntities = (str) => {
+        return str
+            .replace(/&quot;/g, '"')
+            .replace(/&apos;/g, "'")
+            .replace(/&lt;/g, "<")
+            .replace(/&gt;/g, ">")
+            .replace(/&amp;/g, "&");
+    };
+    // 2. ROBUST TEXT ATTRIBUTE PARSING (reasoning, value, full_phrase)
+    // This regex matches: Key = Quote -> Content (lazy) -> Lookahead for (Next Attribute OR End of Tag)
+    // It effectively ignores quotes inside the content during the initial capture.
+    const textAttributeRegex = /(fullPhrase|full_phrase|reasoning|value)\s*=\s*(['"])([\s\S]*?)(?=\s+(?:line_ids|lineIds|timestamps|fileId|file_id|start_page_key|start_pageKey|startPageKey|reasoning|value|full_phrase)|\s*\/?>)/gm;
+    normalized = normalized.replace(textAttributeRegex, (_match, key, openQuote, rawContent) => {
+        let content = rawContent;
+        // The lazy match usually captures the closing quote because the lookahead
+        // starts at the space *after* the attribute. We must strip it.
+        if (content.endsWith(openQuote)) {
+            content = content.slice(0, -1);
+        }
+        // 1. Normalization: Flatten newlines to spaces
+        content = content.replace(/(\r?\n)+/g, " ");
+        // 2. Decode entities to get raw text (e.g., &apos; -> ')
+        content = decodeHtmlEntities(content);
+        // 3. Remove Markdown bold/italic markers often hallucinated by LLMs inside attributes
+        content = content.replace(/(\*|_){2,}/g, "");
+        // 4. Sanitize Quotes:
+        // First, unescape existing backslashed quotes to avoid double escaping (e.g. \\' -> ')
+        content = content.replace(/\\\\'/g, "'");
+        content = content.replace(/\\'/g, "'");
+        content = content.replace(/'/g, "\\'");
+        content = content.replace(/\\\\"/g, '"');
+        content = content.replace(/\\"/g, '"');
+        content = content.replace(/"/g, '\\"');
+        // 5. Remove * from the content, sometimes a md list will really mess things up here so we remove it
+        content = content.replace(/\*/g, ""); //this is a hack to remove the * from the content
+        return `${canonicalizeCiteAttributeKey(key)}='${content}'`;
+    });
+    // 3. ROBUST LINE_ID / TIMESTAMP PARSING
+    // Handles unquoted, single quoted, or double quoted numbers/ranges.
+    // Can handle line_ids appearing anywhere in the tag, not just at the end.
+    normalized = normalized.replace(/(line_ids|lineIds|timestamps)=['"]?([\[\]\(\){}A-Za-z0-9_\-, ]+)['"]?(\s*\/?>|\s+)/gm, (_match, key, rawValue, trailingChars) => {
+        // Clean up the value (remove generic text, keep numbers/separators)
+        let cleanedValue = rawValue.replace(/[A-Za-z\[\]\(\){}]/g, "");
+        // Expand ranges (e.g., "1-3" -> "1,2,3")
+        cleanedValue = cleanedValue.replace(/(\d+)-(\d+)/g, (_rangeMatch, start, end) => {
+            const startNum = parseInt(start, 10);
+            const endNum = parseInt(end, 10);
+            const range = [];
+            // Handle ascending range
+            if (startNum <= endNum) {
+                for (let i = startNum; i <= endNum; i++) {
+                    range.push(i);
+                }
+            }
+            else {
+                // Fallback for weird descending ranges or just return start
+                range.push(startNum);
+            }
+            return range.join(",");
+        });
+        // Normalize commas
+        cleanedValue = cleanedValue.replace(/,+/g, ",").replace(/^,|,$/g, "");
+        // Return standardized format: key='value' + preserved trailing characters (space or />)
+        return `${canonicalizeCiteAttributeKey(key)}='${cleanedValue}'${trailingChars}`;
+    });
+    // 4. Re-order <cite ... /> attributes to match the strict parsing expectations in `citationParser.ts`
+    // (the parser uses regexes that assume a canonical attribute order).
+    const reorderCiteTagAttributes = (tag) => {
+        // Match both single-quoted and double-quoted attributes
+        const attrRegex = /([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(['"])((?:[^'"\\\n]|\\.)*)(?:\2)/g;
+        const attrs = {};
+        let match;
+        while ((match = attrRegex.exec(tag))) {
+            const rawKey = match[1];
+            const value = match[3]; // match[2] is the quote character
+            const key = canonicalizeCiteAttributeKey(rawKey);
+            attrs[key] = value;
+        }
+        // If we didn't find any parsable attrs, don't touch the tag.
+        const keys = Object.keys(attrs);
+        if (keys.length === 0)
+            return tag;
+        const hasTimestamps = typeof attrs.timestamps === "string" && attrs.timestamps.length > 0;
+        const startPageKeys = keys.filter(k => k.startsWith("start_page"));
+        const ordered = [];
+        // Shared first
+        if (attrs.file_id)
+            ordered.push("file_id");
+        if (hasTimestamps) {
+            // AV citations: fileId, full_phrase, timestamps, (optional reasoning/value), then any extras
+            if (attrs.full_phrase)
+                ordered.push("full_phrase");
+            ordered.push("timestamps");
+        }
+        else {
+            // Document citations: fileId, start_page*, full_phrase, line_ids, (optional reasoning/value), then any extras
+            if (startPageKeys.includes("start_page_key"))
+                ordered.push("start_page_key");
+            startPageKeys
+                .filter(k => k !== "start_page_key")
+                .sort()
+                .forEach(k => ordered.push(k));
+            if (attrs.full_phrase)
+                ordered.push("full_phrase");
+            if (attrs.line_ids)
+                ordered.push("line_ids");
+        }
+        // Optional attrs supported by the parser (but not required)
+        if (attrs.reasoning)
+            ordered.push("reasoning");
+        if (attrs.value)
+            ordered.push("value");
+        // Any remaining attributes, stable + deterministic (alpha)
+        const used = new Set(ordered);
+        keys
+            .filter(k => !used.has(k))
+            .sort()
+            .forEach(k => ordered.push(k));
+        const rebuiltAttrs = ordered.map(k => `${k}='${attrs[k]}'`).join(" ");
+        return `<cite ${rebuiltAttrs} />`;
+    };
+    normalized = normalized.replace(/<cite\b[\s\S]*?\/>/gm, tag => reorderCiteTagAttributes(tag));
+    return normalized;
+};

package/lib/parsing/parseCitation.d.ts

@@ -0,0 +1,79 @@
+import { type FoundHighlightLocation } from "../types/foundHighlight.js";
+import { type Citation, type CitationStatus } from "../types/citation.js";
+/**
+ * Calculates the verification status of a citation based on the found highlight and search state.
+ *
+ * @param foundHighlight - The found highlight location, or null/undefined if not found
+ * @returns An object containing boolean flags for verification status
+ */
+export declare function getCitationStatus(foundHighlight: FoundHighlightLocation | null | undefined): CitationStatus;
+export declare const parseCitation: (fragment: string, mdAttachmentId?: string | null, citationCounterRef?: any | null, isVerbose?: boolean) => {
+    beforeCite: string;
+    afterCite: string;
+    citation: Citation;
+};
+/**
+ * Extracts all citations from LLM output.
+ * Supports both XML <cite ... /> tags (embedded in strings/markdown) and JSON-based citation formats.
+ *
+ * For object input:
+ * - Traverses the object looking for `citation` or `citations` properties matching JSON format
+ * - Also stringifies the object to find embedded XML citations in markdown content
+ *
+ * @param llmOutput - The LLM output (string or object)
+ * @returns Dictionary of parsed Citation objects keyed by citation key
+ */
+export declare const getAllCitationsFromLlmOutput: (llmOutput: any) => {
+    [key: string]: Citation;
+};
+/**
+ * Groups citations by their fileId for multi-file verification scenarios.
+ * This is useful when you have citations from multiple files and need to
+ * verify them against their respective source documents.
+ *
+ * @param citations - Array of Citation objects or a dictionary of citations
+ * @returns Map of fileId to dictionary of citations from that file
+ *
+ * @example
+ * ```typescript
+ * const citations = getAllCitationsFromLlmOutput(response.content);
+ * const citationsByFile = groupCitationsByFileId(citations);
+ *
+ * // Verify citations for each file
+ * for (const [fileId, fileCitations] of citationsByFile) {
+ *   const verified = await dc.verifyCitations(fileId, fileCitations);
+ *   // Process verification results...
+ * }
+ * ```
+ */
+export declare function groupCitationsByFileId(citations: Citation[] | {
+    [key: string]: Citation;
+}): Map<string, {
+    [key: string]: Citation;
+}>;
+/**
+ * Groups citations by their fileId and returns as a plain object.
+ * Alternative to groupCitationsByFileId that returns a plain object instead of a Map.
+ *
+ * @param citations - Array of Citation objects or a dictionary of citations
+ * @returns Object with fileId keys mapping to citation dictionaries
+ *
+ * @example
+ * ```typescript
+ * const citations = getAllCitationsFromLlmOutput(response.content);
+ * const citationsByFile = groupCitationsByFileIdObject(citations);
+ *
+ * // Verify citations for each file using Promise.all
+ * const verificationPromises = Object.entries(citationsByFile).map(
+ *   ([fileId, fileCitations]) => dc.verifyCitations(fileId, fileCitations)
+ * );
+ * const results = await Promise.all(verificationPromises);
+ * ```
+ */
+export declare function groupCitationsByFileIdObject(citations: Citation[] | {
+    [key: string]: Citation;
+}): {
+    [fileId: string]: {
+        [key: string]: Citation;
+    };
+};