@gpdoc/filekit 1.0.0

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@gpdoc/filekit",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Headless GPDoc file format detection, validation, and text conversion",
+  "repository": "https://github.com/repetere/gpdoc.git",
+  "files": [
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": "./src/index.js",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "fflate": "^0.8.2",
+    "js-yaml": "^4.1.0",
+    "marked": "^12.0.0"
+  }
+}

package/src/index.js

@@ -0,0 +1,378 @@
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate';
+import yaml from 'js-yaml';
+import { marked } from 'marked';
+
+const MARKDOWN_EXTENSIONS = new Set(['.md', '.markdown', '.mdown', '.gpdoc.md']);
+const HTML_EXTENSIONS = new Set(['.html', '.htm']);
+const TEXT_EXTENSIONS = new Set(['.txt', '.text']);
+const UNSUPPORTED_BINARY_EXTENSIONS = new Set(['.pdf', '.rtf']);
+const VALID_FILETYPES = new Set(['document', 'notebook', 'slides', 'script', 'spreadsheet', 'drawing']);
+const FRONT_MATTER = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
+const MAX_URL_RESPONSE_BYTES = 10 * 1024 * 1024;
+const URL_TIMEOUT_MS = 15_000;
+
+export class FileKitError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+  }
+}
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+    && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null);
+}
+
+function isSafeJsonValue(value, depth = 0) {
+  if (depth > 40) return false;
+  if (value === null || ['string', 'boolean'].includes(typeof value)) return true;
+  if (typeof value === 'number') return Number.isFinite(value);
+  if (Array.isArray(value)) return value.length <= 10_000 && value.every((entry) => isSafeJsonValue(entry, depth + 1));
+  if (!isPlainObject(value)) return false;
+  return Object.entries(value).every(([key, entry]) => !['__proto__', 'prototype', 'constructor'].includes(key)
+    && isSafeJsonValue(entry, depth + 1));
+}
+
+function clone(value) {
+  return JSON.parse(JSON.stringify(value));
+}
+
+function isManagedMetadata(value) {
+  return isPlainObject(value)
+    && typeof value.id === 'string' && value.id.length > 0
+    && typeof value.filename === 'string' && value.filename.length > 0
+    && typeof value.filetype === 'string' && VALID_FILETYPES.has(value.filetype)
+    && isSafeJsonValue(value);
+}
+
+function filenameTitle(filePath) {
+  const name = path.basename(filePath || 'Untitled').replace(/(?:\.gpdoc)?\.[^.]+$/i, '');
+  return name || 'Untitled';
+}
+
+function decodeEntities(value) {
+  return String(value)
+    .replace(/&#(x?[0-9a-f]+);/gi, (_match, encoded) => String.fromCodePoint(Number.parseInt(encoded, encoded[0].toLowerCase() === 'x' ? 16 : 10)))
+    .replace(/&(nbsp|amp|lt|gt|quot|apos|#39);/gi, (_match, name) => ({
+      nbsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", '#39': "'",
+    })[name.toLowerCase()] || _match);
+}
+
+function normalizeBytes(input) {
+  if (input instanceof Uint8Array) return input;
+  if (input instanceof ArrayBuffer) return new Uint8Array(input);
+  if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
+  throw new FileKitError('INVALID_BINARY', 'Expected binary file data.');
+}
+
+function decodeXml(value) {
+  return decodeEntities(String(value)
+    .replace(/&quot;/gi, '"')
+    .replace(/&apos;/gi, "'"));
+}
+
+function escapeXml(value) {
+  return String(value).replace(/[&<>"']/g, (character) => ({
+    '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;',
+  })[character]);
+}
+
+function extractDocxMarkdown(input) {
+  let entries;
+  try {
+    entries = unzipSync(normalizeBytes(input));
+  } catch {
+    throw new FileKitError('INVALID_BINARY', 'DOCX input is not a readable OpenXML package.');
+  }
+  const documentXml = entries['word/document.xml'];
+  if (!documentXml) throw new FileKitError('INVALID_BINARY', 'DOCX input does not contain word/document.xml.');
+  const paragraphs = strFromU8(documentXml).match(/<w:p(?:\s[^>]*)?>[\s\S]*?<\/w:p>/gi) || [];
+  return paragraphs.map((paragraph) => {
+    const style = paragraph.match(/<w:pStyle[^>]*w:val="([^"]+)"/i)?.[1] || '';
+    const text = decodeXml(paragraph
+      .replace(/<w:tab\s*\/?>/gi, '\t')
+      .replace(/<w:br\s*\/?>/gi, '\n')
+      .match(/<w:t(?:\s[^>]*)?>([\s\S]*?)<\/w:t>/gi)?.map((part) => part.replace(/^<w:t(?:\s[^>]*)?>|<\/w:t>$/gi, '')).join('') || '')
+      .replace(/\s+\n/g, '\n').trim();
+    if (!text) return '';
+    const heading = style.match(/heading\s*([1-6])/i)?.[1];
+    return heading ? `${'#'.repeat(Number(heading))} ${text}` : text;
+  }).filter(Boolean).join('\n\n');
+}
+
+function extractPptxMarkdown(input) {
+  let entries;
+  try {
+    entries = unzipSync(normalizeBytes(input));
+  } catch {
+    throw new FileKitError('INVALID_BINARY', 'PPTX input is not a readable OpenXML package.');
+  }
+  const slidePaths = Object.keys(entries).filter((entry) => /^ppt\/slides\/slide\d+\.xml$/i.test(entry))
+    .sort((a, b) => Number(a.match(/slide(\d+)\.xml/i)?.[1]) - Number(b.match(/slide(\d+)\.xml/i)?.[1]));
+  if (!slidePaths.length) throw new FileKitError('INVALID_BINARY', 'PPTX input does not contain slides.');
+  return slidePaths.map((slidePath, index) => {
+    const xml = strFromU8(entries[slidePath]);
+    const lines = (xml.match(/<a:p(?:\s[^>]*)?>[\s\S]*?<\/a:p>/gi) || []).map((paragraph) =>
+      decodeXml((paragraph.match(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/gi) || [])
+        .map((part) => part.replace(/^<a:t(?:\s[^>]*)?>|<\/a:t>$/gi, '')).join('')).trim()).filter(Boolean);
+    const [title = `Slide ${index + 1}`, ...body] = lines;
+    return `# ${title.replace(/^#{1,6}\s+/, '')}${body.length ? `\n\n${body.join('\n\n')}` : ''}`;
+  }).join('\n\n---\n\n');
+}
+
+function createDocx(markdown, title) {
+  const paragraphs = String(markdown).split(/\n{2,}/).map((paragraph) => paragraph.trim()).filter(Boolean);
+  const body = paragraphs.map((paragraph) => {
+    const heading = paragraph.match(/^(#{1,6})\s+([\s\S]*)$/);
+    const text = markdownToText(heading ? heading[2] : paragraph).replace(/\n/g, ' ');
+    const style = heading ? `<w:pPr><w:pStyle w:val="Heading${heading[1].length}"/></w:pPr>` : '';
+    return `<w:p>${style}<w:r><w:t xml:space="preserve">${escapeXml(text)}</w:t></w:r></w:p>`;
+  }).join('');
+  const contentTypes = `<?xml version="1.0" encoding="UTF-8"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/></Types>`;
+  const relationships = `<?xml version="1.0" encoding="UTF-8"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/></Relationships>`;
+  const document = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>${body || `<w:p><w:r><w:t>${escapeXml(title)}</w:t></w:r></w:p>`}<w:sectPr><w:pgSz w:w="12240" w:h="15840"/><w:pgMar w:top="1440" w:right="1440" w:bottom="1440" w:left="1440"/></w:sectPr></w:body></w:document>`;
+  return zipSync({ '[Content_Types].xml': strToU8(contentTypes), '_rels/.rels': strToU8(relationships), 'word/document.xml': strToU8(document) }, { level: 0 });
+}
+
+function createPdf(markdown, title) {
+  const lines = [title, ...markdownToText(markdown).split('\n').filter(Boolean)].flatMap((line) => line.match(/.{1,90}(?:\s|$)|\S+/g) || ['']);
+  const operations = lines.slice(0, 42).map((line, index) => `${index === 0 ? '72 740 Td' : '0 -16 Td'} (${String(line).replace(/[\\()]/g, '\\$&')}) Tj`).join('\n');
+  const stream = `BT\n/F1 12 Tf\n${operations}\nET`;
+  const objects = [
+    '<< /Type /Catalog /Pages 2 0 R >>',
+    '<< /Type /Pages /Kids [3 0 R] /Count 1 >>',
+    '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>',
+    `<< /Length ${Buffer.byteLength(stream, 'utf8')} >>\nstream\n${stream}\nendstream`,
+    '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>',
+  ];
+  let output = '%PDF-1.4\n';
+  const offsets = [0];
+  objects.forEach((object, index) => { offsets.push(Buffer.byteLength(output, 'utf8')); output += `${index + 1} 0 obj\n${object}\nendobj\n`; });
+  const xref = Buffer.byteLength(output, 'utf8');
+  output += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${offsets.slice(1).map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`).join('')}trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
+  return Buffer.from(output, 'utf8');
+}
+
+function inlineHtmlToMarkdown(value) {
+  let markdown = String(value);
+  markdown = markdown.replace(/<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href, label) => `[${inlineHtmlToMarkdown(label).trim() || href}](${decodeEntities(href)})`);
+  markdown = markdown.replace(/<(?:strong|b)\b[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**');
+  markdown = markdown.replace(/<(?:em|i)\b[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*');
+  markdown = markdown.replace(/<(?:del|s|strike)\b[^>]*>([\s\S]*?)<\/(?:del|s|strike)>/gi, '~~$1~~');
+  markdown = markdown.replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, '`$1`');
+  return decodeEntities(markdown.replace(/<[^>]+>/g, ''));
+}
+
+export function htmlToMarkdown(html) {
+  let markdown = String(html || '').replace(/\r\n/g, '\n');
+  markdown = markdown.replace(/<!--[\s\S]*?-->/g, '');
+  markdown = markdown.replace(/<\s*(script|style|noscript)\b[^>]*>[\s\S]*?<\s*\/\s*\1>/gi, '');
+  const body = markdown.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
+  if (body) markdown = body[1];
+  else markdown = markdown
+    .replace(/<head\b[^>]*>[\s\S]*?<\/head>/gi, '')
+    .replace(/<title\b[^>]*>[\s\S]*?<\/title>/gi, '');
+  markdown = markdown.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level, content) => `${'#'.repeat(Number(level))} ${inlineHtmlToMarkdown(content).trim()}\n\n`);
+  markdown = markdown.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, content) => `- ${inlineHtmlToMarkdown(content).trim()}\n`);
+  markdown = markdown.replace(/<\/?(?:ul|ol)\b[^>]*>/gi, '\n');
+  markdown = markdown.replace(/<p\b[^>]*>([\s\S]*?)<\/p>/gi, (_match, content) => `${inlineHtmlToMarkdown(content).trim()}\n\n`);
+  markdown = markdown.replace(/<br\s*\/?>/gi, '\n');
+  markdown = inlineHtmlToMarkdown(markdown);
+  return markdown.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').trim();
+}
+
+function htmlTitle(html, fallback) {
+  const title = String(html).match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+  return title ? decodeEntities(title.replace(/<[^>]+>/g, '')).trim() || fallback : fallback;
+}
+
+export async function readUrlDocument(input) {
+  let url;
+  try {
+    url = new URL(input);
+  } catch {
+    throw new FileKitError('INVALID_URL', 'URL input must be a valid HTTP or HTTPS URL.');
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) {
+    throw new FileKitError('INVALID_URL', 'URL input must use HTTP or HTTPS.');
+  }
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), URL_TIMEOUT_MS);
+  let response;
+  try {
+    response = await fetch(url, { redirect: 'follow', signal: controller.signal });
+  } catch (error) {
+    const message = error?.name === 'AbortError' ? 'Timed out while fetching URL input.' : 'Failed to fetch URL input.';
+    throw new FileKitError('URL_FETCH_FAILED', message);
+  } finally {
+    clearTimeout(timeout);
+  }
+  if (!response.ok) throw new FileKitError('URL_FETCH_FAILED', `URL input returned HTTP ${response.status}.`);
+  const contentType = response.headers.get('content-type')?.toLowerCase() || '';
+  if (!contentType.startsWith('text/html') && !contentType.startsWith('text/plain')) {
+    throw new FileKitError('UNSUPPORTED_URL_CONTENT', 'URL input must return HTML or plain text.');
+  }
+  const declaredSize = Number(response.headers.get('content-length'));
+  if (Number.isFinite(declaredSize) && declaredSize > MAX_URL_RESPONSE_BYTES) {
+    throw new FileKitError('URL_RESPONSE_TOO_LARGE', 'URL input exceeds the 10 MiB response limit.');
+  }
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of response.body || []) {
+    size += chunk.byteLength;
+    if (size > MAX_URL_RESPONSE_BYTES) throw new FileKitError('URL_RESPONSE_TOO_LARGE', 'URL input exceeds the 10 MiB response limit.');
+    chunks.push(chunk);
+  }
+  const source = new TextDecoder().decode(Buffer.concat(chunks));
+  if (contentType.startsWith('text/plain')) {
+    return { format: 'url-text', managed: false, filetype: 'document', title: filenameTitle(url.pathname) || url.hostname, body: source, warnings: [], sourceUrl: url.toString() };
+  }
+  return { format: 'url-html', managed: false, filetype: 'document', title: htmlTitle(source, filenameTitle(url.pathname) || url.hostname), body: htmlToMarkdown(source), warnings: ['HTML conversion may omit source-only styling.'], sourceUrl: url.toString() };
+}
+
+export function detectFormat(filePath) {
+  const lower = String(filePath || '').toLowerCase();
+  if (lower.endsWith('.gpdoc.md')) return 'markdown';
+  const extension = path.extname(lower);
+  if (MARKDOWN_EXTENSIONS.has(extension)) return 'markdown';
+  if (HTML_EXTENSIONS.has(extension)) return 'html';
+  if (TEXT_EXTENSIONS.has(extension)) return 'text';
+  if (extension === '.json') return 'json';
+  if (extension === '.docx') return 'docx';
+  if (extension === '.pptx') return 'pptx';
+  if (UNSUPPORTED_BINARY_EXTENSIONS.has(extension)) throw new FileKitError('UNSUPPORTED_FORMAT', `Unsupported input format: ${extension}.`);
+  throw new FileKitError('UNSUPPORTED_FORMAT', `Unsupported input format: ${extension || 'unknown'}.`);
+}
+
+function parseManagedMarkdown(source) {
+  const match = FRONT_MATTER.exec(source);
+  if (!match) return null;
+  const raw = match[1]?.trim() || '';
+  if (!raw) return null;
+  try {
+    const json = JSON.parse(raw);
+    if (isManagedMetadata(json)) return { metadata: json, body: source.slice(match[0].length), frontMatterFormat: 'json' };
+    if (isPlainObject(json)) throw new FileKitError('INVALID_GPDOC', 'Managed GPDoc metadata must include id, filename, and a supported filetype.');
+  } catch (error) {
+    if (error instanceof FileKitError) throw error;
+  }
+  let yamlRoot;
+  try {
+    yamlRoot = yaml.load(raw, { schema: yaml.JSON_SCHEMA, json: true });
+  } catch {
+    if (/gpdoc_metadata\s*:/.test(raw)) {
+      throw new FileKitError('INVALID_GPDOC', 'Managed GPDoc metadata must include id, filename, and a supported filetype.');
+    }
+    return null;
+  }
+  if (!isPlainObject(yamlRoot) || !Object.hasOwn(yamlRoot, 'gpdoc_metadata')) return null;
+  if (!isManagedMetadata(yamlRoot.gpdoc_metadata)) {
+    throw new FileKitError('INVALID_GPDOC', 'Managed GPDoc metadata must include id, filename, and a supported filetype.');
+  }
+  return { metadata: clone(yamlRoot.gpdoc_metadata), body: source.slice(match[0].length), frontMatterFormat: 'yaml' };
+}
+
+function parseGpdocJson(source) {
+  let json;
+  try {
+    json = JSON.parse(source);
+  } catch {
+    throw new FileKitError('INVALID_GPDOC', 'GPDoc JSON must be valid JSON.');
+  }
+  if (!isManagedMetadata(json) || !isPlainObject(json.content)) {
+    throw new FileKitError('INVALID_GPDOC', 'Managed GPDoc metadata must include id, filename, and a supported filetype.');
+  }
+  if (typeof json.content.document !== 'string') {
+    throw new FileKitError('INVALID_GPDOC', 'This CLI release supports document content only.');
+  }
+  return { metadata: clone(json), body: json.content.document };
+}
+
+export function readDocument(source, filePath = '') {
+  const format = detectFormat(filePath);
+  if (format === 'docx') {
+    return { format, managed: false, filetype: 'document', title: filenameTitle(filePath), body: extractDocxMarkdown(source), warnings: ['DOCX conversion preserves text and headings, not full Word formatting.'] };
+  }
+  if (format === 'pptx') {
+    return { format, managed: false, filetype: 'document', title: filenameTitle(filePath), body: extractPptxMarkdown(source), warnings: ['PPTX conversion preserves slide text, not layout or speaker notes.'] };
+  }
+  if (format === 'html') {
+    return { format, managed: false, filetype: 'document', title: filenameTitle(filePath), body: htmlToMarkdown(source), warnings: ['HTML conversion may omit source-only styling.'] };
+  }
+  if (format === 'text') {
+    return { format, managed: false, filetype: 'document', title: filenameTitle(filePath), body: String(source), warnings: [] };
+  }
+  if (format === 'json') {
+    const parsed = parseGpdocJson(source);
+    return { format: 'gpdoc-json', managed: true, filetype: parsed.metadata.filetype, title: parsed.metadata.filename, body: parsed.body, metadata: parsed.metadata, warnings: [] };
+  }
+  const parsed = parseManagedMarkdown(String(source));
+  if (parsed) {
+    return { format: 'gpdoc-markdown', managed: true, filetype: parsed.metadata.filetype, title: parsed.metadata.filename, body: parsed.body, metadata: parsed.metadata, warnings: [] };
+  }
+  return { format: 'markdown', managed: false, filetype: 'document', title: filenameTitle(filePath), body: String(source), warnings: [] };
+}
+
+export function createManagedDocument(title = 'Untitled', body = '') {
+  const now = new Date().toISOString();
+  return {
+    id: `file_${randomUUID()}`,
+    filename: title,
+    filetype: 'document',
+    created: now,
+    updated: now,
+    content: { document: String(body) },
+  };
+}
+
+export function serializeManagedMarkdown(metadata, body) {
+  const root = { gpdoc_metadata: { ...metadata, content: undefined } };
+  root.gpdoc_metadata.content = { document: String(body) };
+  return `---\n${yaml.dump(root, { schema: yaml.JSON_SCHEMA, noRefs: true, lineWidth: -1 }).trimEnd()}\n---\n${String(body)}`;
+}
+
+function markdownToText(markdown) {
+  return String(markdown)
+    .replace(/```[\s\S]*?```/g, (block) => block.replace(/^```[^\n]*\n?|```$/g, ''))
+    .replace(/!?(\[([^\]]*)\]\([^)]*\))/g, '$2')
+    .replace(/[*_~`>#]/g, '')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+export function convertDocument(document, targetFormat) {
+  const target = String(targetFormat).toLowerCase();
+  const metadata = document.managed ? clone(document.metadata) : createManagedDocument(document.title, document.body);
+  metadata.content = { document: document.body };
+  if (target === 'gpdoc') return serializeManagedMarkdown(metadata, document.body);
+  if (target === 'markdown' || target === 'md') return document.body;
+  if (target === 'html') return `<!doctype html>\n<html><head><meta charset="utf-8"><title>${escapeHtml(document.title)}</title></head><body>\n${marked.parse(document.body)}\n</body></html>\n`;
+  if (target === 'text' || target === 'txt') return markdownToText(document.body);
+  if (target === 'json' || target === 'gpdoc-json') return `${JSON.stringify(metadata, null, 2)}\n`;
+  if (target === 'docx') return createDocx(document.body, document.title);
+  if (target === 'pdf') return createPdf(document.body, document.title);
+  throw new FileKitError('UNSUPPORTED_OUTPUT', `Unsupported output format: ${targetFormat}.`);
+}
+
+function escapeHtml(value) {
+  return String(value).replace(/[&<>"']/g, (character) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[character]);
+}
+
+export function inspectDocument(document, filePath) {
+  return {
+    format: document.format,
+    managed: document.managed,
+    filetype: document.filetype,
+    title: document.title,
+    path: filePath,
+  };
+}
+
+export function validateDocument(document) {
+  if (document.managed && !isManagedMetadata(document.metadata)) {
+    throw new FileKitError('INVALID_GPDOC', 'Managed GPDoc metadata must include id, filename, and a supported filetype.');
+  }
+  return true;
+}