@pdpp/local-collector 0.0.0

package/dist/polyfill-connectors/connectors/claude_code/parsers.js

@@ -0,0 +1,224 @@
+import { safeTextPreview } from "../../src/safe-text-preview.js";
+export const SHORT_PREVIEW_CHARS = 300;
+export const ATTACHMENT_PREVIEW_CHARS = 500;
+export const TOOL_RESULT_PREVIEW_CHARS = 500;
+export const MESSAGE_CONTENT_PREVIEW_CHARS = 5000;
+export const SKILL_BODY_MAX_CHARS = 20_000;
+export const LINE_PROGRESS_INTERVAL = 2000;
+export const BYTES_PER_MB = 1024 * 1024;
+export const SESSION_DIR_PREFIX_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-/;
+const CLAUDE_FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
+const CLAUDE_FM_LINE_RE = /^([A-Za-z0-9_-]+)\s*:\s*(.*)$/;
+const CLAUDE_FM_COMMENT_RE = /^\s*#/;
+const CLAUDE_FM_INDENT_RE = /^\s+\S/;
+const CLAUDE_FM_LEADING_WS_RE = /^\s+/;
+const CLAUDE_FM_QUOTED_DOUBLE_RE = /^"([\s\S]*)"$/;
+const CLAUDE_FM_QUOTED_SINGLE_RE = /^'([\s\S]*)'$/;
+const CLAUDE_FM_COLLAPSE_WS_RE = /\s+/g;
+const CLAUDE_FM_LINE_SPLIT_RE = /\r?\n/;
+const CLAUDE_MD_SUFFIX_RE = /\.md$/i;
+export function textPreview(s, max = SHORT_PREVIEW_CHARS) {
+    return safeTextPreview(s, max).preview;
+}
+export function truncateBody(body, max = SKILL_BODY_MAX_CHARS) {
+    return body.length > max ? body.slice(0, max) : body;
+}
+function extractFromArrayPart(p) {
+    if (typeof p === "string") {
+        return p;
+    }
+    const part = p;
+    if (part?.type === "text" && part.text) {
+        return part.text;
+    }
+    if (part?.type === "tool_use") {
+        return `[tool_use: ${part.name || "unknown"}]`;
+    }
+    if (part?.type === "tool_result") {
+        return "[tool_result]";
+    }
+    return "";
+}
+function extractFromArray(arr) {
+    const parts = arr.map(extractFromArrayPart).filter(Boolean);
+    return parts.join("\n") || null;
+}
+function extractFromObject(obj) {
+    if (obj.content) {
+        return extractContent(obj.content);
+    }
+    if (typeof obj.text === "string") {
+        return obj.text;
+    }
+    return null;
+}
+export function extractContent(obj) {
+    if (!obj) {
+        return null;
+    }
+    if (typeof obj === "string") {
+        return obj;
+    }
+    if (Array.isArray(obj)) {
+        return extractFromArray(obj);
+    }
+    if (typeof obj === "object") {
+        return extractFromObject(obj);
+    }
+    return null;
+}
+function stripQuotes(value) {
+    return value.replace(CLAUDE_FM_QUOTED_DOUBLE_RE, "$1").replace(CLAUDE_FM_QUOTED_SINGLE_RE, "$1").trim();
+}
+function isBlockScalar(value) {
+    return value === ">" || value === "|" || value === ">-" || value === "|-";
+}
+function readBlockScalar(lines, startIdx, marker) {
+    const folded = marker.startsWith(">");
+    const collected = [];
+    let i = startIdx;
+    while (i < lines.length) {
+        const next = lines[i] ?? "";
+        if (CLAUDE_FM_INDENT_RE.test(next) || next === "") {
+            collected.push(next.replace(CLAUDE_FM_LEADING_WS_RE, ""));
+            i++;
+        }
+        else {
+            break;
+        }
+    }
+    const value = folded
+        ? collected.join(" ").replace(CLAUDE_FM_COLLAPSE_WS_RE, " ").trim()
+        : collected.join("\n").trim();
+    return { nextIndex: i, value };
+}
+export function parseFrontmatter(text) {
+    if (typeof text !== "string") {
+        return { frontmatter: {}, body: text || "" };
+    }
+    const m = CLAUDE_FRONTMATTER_RE.exec(text);
+    if (!m) {
+        return { frontmatter: {}, body: text };
+    }
+    const rawFm = m[1] ?? "";
+    const body = m[2] ?? "";
+    const frontmatter = {};
+    const lines = rawFm.split(CLAUDE_FM_LINE_SPLIT_RE);
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i] ?? "";
+        if (!line.trim() || CLAUDE_FM_COMMENT_RE.test(line)) {
+            i++;
+            continue;
+        }
+        const kv = CLAUDE_FM_LINE_RE.exec(line);
+        if (!kv) {
+            i++;
+            continue;
+        }
+        const key = kv[1] ?? "";
+        const rawValue = kv[2] ?? "";
+        if (isBlockScalar(rawValue)) {
+            const { nextIndex, value } = readBlockScalar(lines, i + 1, rawValue);
+            frontmatter[key] = value;
+            i = nextIndex;
+        }
+        else {
+            frontmatter[key] = stripQuotes(rawValue);
+            i++;
+        }
+    }
+    return { frontmatter, body };
+}
+export function makeEmptySessionAccumulator(id, projectPath) {
+    return {
+        id,
+        project_path: projectPath,
+        cwd: null,
+        git_branch: null,
+        version: null,
+        started_at: null,
+        last_event_at: null,
+        message_count: 0,
+        user_type: null,
+        entrypoint: null,
+    };
+}
+export function mergeSessionObservations(acc, obs) {
+    if (obs.cwd) {
+        acc.cwd = obs.cwd;
+    }
+    if (obs.gitBranch) {
+        acc.git_branch = obs.gitBranch;
+    }
+    if (obs.version) {
+        acc.version = obs.version;
+    }
+    if (obs.userType) {
+        acc.user_type = obs.userType;
+    }
+    if (obs.entrypoint) {
+        acc.entrypoint = obs.entrypoint;
+    }
+}
+export function widenSessionTimeRange(acc, firstTimestamp, lastTimestamp) {
+    if (firstTimestamp && (!acc.started_at || firstTimestamp < acc.started_at)) {
+        acc.started_at = firstTimestamp;
+    }
+    if (lastTimestamp && (!acc.last_event_at || lastTimestamp > acc.last_event_at)) {
+        acc.last_event_at = lastTimestamp;
+    }
+}
+export function buildSkillRecord(args) {
+    return {
+        id: `skills:${args.name}`,
+        name: args.frontmatter.name || args.name,
+        description: args.frontmatter.description || null,
+        source: "user",
+        path: args.path,
+        content: truncateBody(args.body),
+        frontmatter: args.frontmatter,
+        mtime_epoch: Math.floor(args.mtimeMs / 1000),
+    };
+}
+export function buildMemoryNoteRecord(args) {
+    const fallbackName = args.relPath.replace(CLAUDE_MD_SUFFIX_RE, "");
+    return {
+        id: `memory_notes:${args.projectDir}/${args.relPath}`,
+        project_path: args.projectDir,
+        note_path: args.relPath,
+        name: args.frontmatter.name || args.frontmatter.title || fallbackName,
+        description: args.frontmatter.description || null,
+        path: args.path,
+        content: truncateBody(args.body),
+        frontmatter: args.frontmatter,
+        mtime_epoch: Math.floor(args.mtimeMs / 1000),
+    };
+}
+export function buildSlashCommandRecord(args) {
+    return {
+        id: `commands:${args.idPath}`,
+        name: args.frontmatter.name || args.base,
+        description: args.frontmatter.description || null,
+        path: args.path,
+        content: truncateBody(args.body),
+        frontmatter: args.frontmatter,
+        mtime_epoch: Math.floor(args.mtimeMs / 1000),
+    };
+}
+export function applyProjectDirScope(dirs, include, exclude) {
+    let out = dirs;
+    if (include.length) {
+        out = out.filter((d) => include.some((s) => d.includes(s)));
+    }
+    if (exclude.length) {
+        out = out.filter((d) => !exclude.some((s) => d.includes(s)));
+    }
+    return out;
+}
+export function parseCsvEnv(value) {
+    return (value || "")
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+}

package/dist/polyfill-connectors/connectors/claude_code/schemas.js

@@ -0,0 +1,120 @@
+import { z } from "zod";
+import { pdppSafeText } from "../../src/pdpp-safe-text.js";
+import { PDPP_PREVIEW_MAX_CHARS, safeTextPreview } from "../../src/safe-text-preview.js";
+import { makeValidateRecord } from "../../src/schema-registry.js";
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const ISO_Z_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
+const uuidSchema = z.string().regex(UUID_RE, "must be valid UUID");
+const isoDateTimeSchema = z.string().regex(ISO_Z_RE, "must be ISO-8601 with millis and Z suffix").nullable();
+const stringMaxSchema = (max) => pdppSafeText.max(max).nullable();
+const pathSchema = pdppSafeText.max(2048).nullable();
+export const sessionsSchema = z.object({
+    id: uuidSchema,
+    project_path: pdppSafeText,
+    cwd: pathSchema,
+    git_branch: stringMaxSchema(256),
+    version: stringMaxSchema(64),
+    started_at: isoDateTimeSchema,
+    last_event_at: isoDateTimeSchema,
+    message_count: z.number().int().min(0).nullable(),
+    user_type: stringMaxSchema(40),
+    entrypoint: stringMaxSchema(256),
+});
+export const messagesSchema = z.object({
+    id: uuidSchema,
+    session_id: uuidSchema,
+    parent_uuid: uuidSchema.nullable(),
+    role: stringMaxSchema(64),
+    type: stringMaxSchema(64),
+    content: pdppSafeText.max(10_000_000).nullable(),
+    timestamp: isoDateTimeSchema,
+    is_sidechain: z.boolean(),
+    user_type: stringMaxSchema(40),
+    agent_id: stringMaxSchema(256).nullable(),
+});
+export const attachmentsSchema = z.object({
+    id: pdppSafeText.min(1).max(2048),
+    session_id: uuidSchema,
+    parent_uuid: uuidSchema.nullable(),
+    event_type: stringMaxSchema(64),
+    hook_name: stringMaxSchema(256),
+    tool_use_id: stringMaxSchema(256),
+    content_preview: z
+        .string()
+        .max(PDPP_PREVIEW_MAX_CHARS + 1)
+        .refine((val) => {
+        const result = safeTextPreview(val, PDPP_PREVIEW_MAX_CHARS);
+        return result.kind === "text" || result.kind === "empty";
+    }, "content_preview contains forbidden control characters")
+        .nullable(),
+    content_binary_reason: pdppSafeText.max(200).nullable().optional(),
+    content_bytes: z.number().int().min(0).nullable(),
+    timestamp: isoDateTimeSchema,
+});
+export const skillsSchema = z.object({
+    id: pdppSafeText,
+    name: stringMaxSchema(256),
+    description: stringMaxSchema(2048),
+    source: stringMaxSchema(64),
+    path: pathSchema,
+    content: pdppSafeText.max(10_000_000).nullable(),
+    frontmatter: z.record(z.string(), z.unknown()).nullable(),
+    mtime_epoch: z.number().nullable(),
+});
+export const memoryNotesSchema = z.object({
+    id: pdppSafeText,
+    project_path: pdppSafeText,
+    note_path: pdppSafeText,
+    name: stringMaxSchema(256),
+    description: stringMaxSchema(2048),
+    path: pathSchema,
+    content: pdppSafeText.max(10_000_000).nullable(),
+    frontmatter: z.record(z.string(), z.unknown()).nullable(),
+    mtime_epoch: z.number().nullable(),
+});
+export const slashCommandsSchema = z.object({
+    id: pdppSafeText,
+    name: stringMaxSchema(256),
+    description: stringMaxSchema(2048),
+    path: pathSchema,
+    content: pdppSafeText.max(10_000_000).nullable(),
+    frontmatter: z.record(z.string(), z.unknown()).nullable(),
+    mtime_epoch: z.number().nullable(),
+});
+const inventoryClassificationSchema = z.enum(["inventory_only", "defer"]);
+const inventoryTypeSchema = z.enum(["directory", "file", "missing", "other"]);
+const coverageStatusSchema = z.enum(["collected", "inventory_only", "excluded", "deferred", "missing", "unsupported"]);
+export const inventorySchema = z.object({
+    id: pdppSafeText,
+    store: pdppSafeText,
+    relative_path: pdppSafeText.max(2048),
+    path_hash: z.string().regex(/^[a-f0-9]{64}$/),
+    type: inventoryTypeSchema,
+    size_bytes: z.number().int().min(0).nullable(),
+    mtime_epoch: z.number().int().min(0).nullable(),
+    classification: inventoryClassificationSchema,
+    reason: pdppSafeText.max(512),
+});
+export const coverageDiagnosticsSchema = z.object({
+    id: pdppSafeText,
+    store: pdppSafeText,
+    stream: pdppSafeText.nullable(),
+    status: coverageStatusSchema,
+    reason: pdppSafeText.max(512),
+});
+export const SCHEMAS = {
+    sessions: sessionsSchema,
+    messages: messagesSchema,
+    attachments: attachmentsSchema,
+    skills: skillsSchema,
+    memory_notes: memoryNotesSchema,
+    slash_commands: slashCommandsSchema,
+    file_history: inventorySchema,
+    debug_artifacts: inventorySchema,
+    downloads: inventorySchema,
+    cache_inventory: inventorySchema,
+    backup_inventory: inventorySchema,
+    config_inventory: inventorySchema,
+    coverage_diagnostics: coverageDiagnosticsSchema,
+};
+export const validateRecord = makeValidateRecord(SCHEMAS);

package/dist/polyfill-connectors/connectors/claude_code/types.js

@@ -0,0 +1 @@
+export {};