@bendyline/squisq-formats 1.3.1 → 1.4.0

package/src/__tests__/roundTripMatrix.fixtures.ts

@@ -0,0 +1,86 @@
+import { parseMarkdown, type MarkdownDocument } from '@bendyline/squisq/markdown';
+
+export interface RoundTripFixture {
+  name: string;
+  markdown: string;
+  doc: MarkdownDocument;
+  keyPhrases: string[];
+  headingTexts: string[];
+}
+
+function makeFixture(
+  name: string,
+  markdown: string,
+  keyPhrases: string[],
+  headingTexts: string[],
+): RoundTripFixture {
+  return {
+    name,
+    markdown,
+    doc: parseMarkdown(markdown),
+    keyPhrases,
+    headingTexts,
+  };
+}
+
+export const ROUNDTRIP_FIXTURES = {
+  story: makeFixture(
+    'story',
+    `# River Journal
+
+The field team walked the north bank and recorded water conditions.
+
+## Morning Snapshot
+
+- Water level is steady.
+- Team status is ready.
+- Safety checks are complete.
+
+## Notes
+
+The crew published an update and linked the [full report](https://example.com/report).
+`,
+    ['river journal', 'north bank', 'water level', 'team status', 'full report'],
+    ['River Journal', 'Morning Snapshot', 'Notes'],
+  ),
+
+  mixed: makeFixture(
+    'mixed',
+    `# Ops Weekly
+
+This week we tracked **launch readiness**, *quality signals*, and risk burn-down.
+
+## Checklist
+
+1. Confirm deployment window.
+2. Validate rollback script.
+3. Notify stakeholders.
+
+## Outcome
+
+Launch readiness improved and support tickets declined.
+`,
+    [
+      'ops weekly',
+      'launch readiness',
+      'quality signals',
+      'rollback script',
+      'support tickets declined',
+    ],
+    ['Ops Weekly', 'Checklist', 'Outcome'],
+  ),
+
+  table: makeFixture(
+    'table',
+    `# Metrics
+
+| Metric | Value |
+| --- | --- |
+| Throughput | 120 req/s |
+| Error Rate | 0.2% |
+| P95 | 180ms |
+`,
+    ['metrics', 'throughput', '120 req/s', 'error rate', 'p95'],
+    ['Metrics'],
+  ),
+};

package/src/__tests__/roundTripMatrix.helpers.ts

@@ -0,0 +1,154 @@
+import { expect } from 'vitest';
+import type {
+  MarkdownDocument,
+  MarkdownNode,
+  MarkdownBlockNode,
+  MarkdownHeading,
+} from '@bendyline/squisq/markdown';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
+
+function collectText(node: unknown, out: string[]): void {
+  if (!isRecord(node)) return;
+
+  const maybeType = node.type;
+  if (typeof maybeType === 'string') {
+    if (maybeType === 'text' && typeof node.value === 'string') {
+      out.push(node.value);
+    }
+    if ((maybeType === 'code' || maybeType === 'inlineCode') && typeof node.value === 'string') {
+      out.push(node.value);
+    }
+    if (maybeType === 'link' && typeof node.url === 'string') {
+      out.push(node.url);
+    }
+    if (maybeType === 'image') {
+      if (typeof node.alt === 'string') out.push(node.alt);
+      if (typeof node.url === 'string') out.push(node.url);
+    }
+  }
+
+  const children = node.children;
+  if (Array.isArray(children)) {
+    for (const child of children) collectText(child, out);
+  }
+}
+
+function normalizeText(input: string): string {
+  return input
+    .toLowerCase()
+    .replace(/\r\n/g, '\n')
+    .replace(/[^a-z0-9%/.:\-\s]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function compactText(input: string): string {
+  return normalizeText(input).replace(/\s+/g, '');
+}
+
+function extractDocumentText(doc: MarkdownDocument): string {
+  const out: string[] = [];
+  collectText(doc, out);
+  return normalizeText(out.join(' '));
+}
+
+function extractHeadingTexts(doc: MarkdownDocument): string[] {
+  return doc.children
+    .filter((block): block is MarkdownHeading => block.type === 'heading')
+    .map((heading) => {
+      const out: string[] = [];
+      collectText(heading, out);
+      return normalizeText(out.join(' '));
+    })
+    .filter(Boolean);
+}
+
+function containsNodeType(node: unknown, wantedType: string): boolean {
+  if (!isRecord(node)) return false;
+  if (node.type === wantedType) return true;
+  const children = node.children;
+  if (!Array.isArray(children)) return false;
+  return children.some((child) => containsNodeType(child, wantedType));
+}
+
+function retainRatio(source: string, roundTrip: string): number {
+  const sourceTokens = source.split(' ').filter((t) => t.length >= 4);
+  if (sourceTokens.length === 0) return 1;
+
+  const targetSet = new Set(roundTrip.split(' ').filter((t) => t.length >= 4));
+  let matched = 0;
+  for (const token of sourceTokens) {
+    if (targetSet.has(token)) matched++;
+  }
+  return matched / sourceTokens.length;
+}
+
+export interface HybridRoundTripAssertions {
+  source: MarkdownDocument;
+  roundTrip: MarkdownDocument;
+  keyPhrases: string[];
+  headingTexts?: string[];
+  requireHeadingOrder?: boolean;
+  requireListSurvival?: boolean;
+  requireTableSurvival?: boolean;
+  minRetainRatio?: number;
+}
+
+export function assertHybridRoundTrip(args: HybridRoundTripAssertions): void {
+  const sourceText = extractDocumentText(args.source);
+  const roundTripText = extractDocumentText(args.roundTrip);
+
+  expect(roundTripText.length).toBeGreaterThan(0);
+
+  const minRetainRatio = args.minRetainRatio ?? 0.5;
+  expect(retainRatio(sourceText, roundTripText)).toBeGreaterThanOrEqual(minRetainRatio);
+
+  for (const phrase of args.keyPhrases) {
+    const normalizedPhrase = normalizeText(phrase);
+    if (roundTripText.includes(normalizedPhrase)) continue;
+    expect(compactText(roundTripText)).toContain(compactText(normalizedPhrase));
+  }
+
+  if (args.requireHeadingOrder && args.headingTexts && args.headingTexts.length > 0) {
+    const normalizedExpected = args.headingTexts.map((h) => normalizeText(h));
+    const actualHeadings = extractHeadingTexts(args.roundTrip);
+
+    let cursor = -1;
+    for (const expectedHeading of normalizedExpected) {
+      const next = actualHeadings.findIndex(
+        (heading, index) => index > cursor && heading.includes(expectedHeading),
+      );
+      expect(next).toBeGreaterThan(cursor);
+      cursor = next;
+    }
+  }
+
+  if (args.requireListSurvival && containsNodeType(args.source, 'list')) {
+    expect(containsNodeType(args.roundTrip, 'list')).toBe(true);
+  }
+
+  if (args.requireTableSurvival && containsNodeType(args.source, 'table')) {
+    expect(containsNodeType(args.roundTrip, 'table')).toBe(true);
+  }
+}
+
+export function extractNormalizedText(doc: MarkdownDocument): string {
+  return extractDocumentText(doc);
+}
+
+export function docHasNodeType(doc: MarkdownDocument, wantedType: MarkdownNode['type']): boolean {
+  return containsNodeType(doc, wantedType);
+}
+
+export function firstHeadingText(doc: MarkdownDocument): string | null {
+  const firstHeading = doc.children.find(
+    (block): block is MarkdownBlockNode & { type: 'heading' } => block.type === 'heading',
+  );
+  if (!firstHeading) return null;
+  const out: string[] = [];
+  collectText(firstHeading, out);
+  return normalizeText(out.join(' '));
+}

package/src/__tests__/roundTripMatrix.test.ts

@@ -0,0 +1,136 @@
+import { describe, expect, it } from 'vitest';
+import JSZip from 'jszip';
+import { markdownToDoc } from '@bendyline/squisq/doc';
+
+import { markdownDocToDocx } from '../docx/export';
+import { docxToMarkdownDoc } from '../docx/import';
+import { markdownDocToPptx } from '../pptx/export';
+import { pptxToMarkdownDoc } from '../pptx/import';
+import { markdownDocToPdf } from '../pdf/export';
+import { pdfToMarkdownDoc } from '../pdf/import';
+import { markdownDocToCsv, csvToMarkdownDoc } from '../csv/index';
+import { markdownDocToEpub } from '../epub/export';
+import { docToHtml } from '../html/index';
+import { markdownDocToXlsx } from '../xlsx/index';
+import { assertHybridRoundTrip, extractNormalizedText } from './roundTripMatrix.helpers';
+import { ROUNDTRIP_FIXTURES } from './roundTripMatrix.fixtures';
+
+const MOCK_PLAYER_SCRIPT = 'var SquisqPlayer={mount:function(){}};';
+
+describe('format content-flow matrix', () => {
+  it('round-trips markdown through DOCX with hybrid equivalence', async () => {
+    const source = ROUNDTRIP_FIXTURES.story;
+    const docx = await markdownDocToDocx(source.doc);
+    const roundTrip = await docxToMarkdownDoc(docx);
+
+    assertHybridRoundTrip({
+      source: source.doc,
+      roundTrip,
+      keyPhrases: source.keyPhrases,
+      headingTexts: source.headingTexts,
+      requireHeadingOrder: true,
+      requireListSurvival: true,
+      minRetainRatio: 0.65,
+    });
+  });
+
+  it('round-trips markdown through PPTX with hybrid equivalence', async () => {
+    const source = ROUNDTRIP_FIXTURES.mixed;
+    const pptx = await markdownDocToPptx(source.doc);
+    const roundTrip = await pptxToMarkdownDoc(pptx);
+
+    assertHybridRoundTrip({
+      source: source.doc,
+      roundTrip,
+      keyPhrases: source.keyPhrases,
+      headingTexts: source.headingTexts,
+      requireHeadingOrder: true,
+      minRetainRatio: 0.5,
+    });
+  });
+
+  it('round-trips markdown through PDF with hybrid equivalence', async () => {
+    const source = ROUNDTRIP_FIXTURES.mixed;
+    const pdf = await markdownDocToPdf(source.doc, { title: 'Ops Weekly' });
+    const roundTrip = await pdfToMarkdownDoc(pdf, { bodyFontSize: 11 });
+
+    assertHybridRoundTrip({
+      source: source.doc,
+      roundTrip,
+      keyPhrases: ['ops weekly', 'launch readiness', 'rollback script', 'support tickets declined'],
+      minRetainRatio: 0.4,
+    });
+  });
+
+  it('round-trips markdown through CSV for table-centric content', async () => {
+    const source = ROUNDTRIP_FIXTURES.table;
+    const csv = markdownDocToCsv(source.doc);
+    const roundTrip = await csvToMarkdownDoc(csv);
+
+    assertHybridRoundTrip({
+      source: source.doc,
+      roundTrip,
+      keyPhrases: ['throughput', '120 req/s', 'error rate', 'p95'],
+      requireTableSurvival: true,
+      minRetainRatio: 0.7,
+    });
+  });
+
+  it('exports markdown to HTML as one-way smoke coverage', async () => {
+    const source = ROUNDTRIP_FIXTURES.story;
+    const doc = markdownToDoc(source.doc, { articleId: 'matrix-story' });
+    const html = docToHtml(doc, {
+      playerScript: MOCK_PLAYER_SCRIPT,
+      title: 'Matrix Story',
+    });
+
+    expect(html).toContain('<!DOCTYPE html>');
+    const normalizedHtml = html.toLowerCase();
+    expect(normalizedHtml).toContain('river journal');
+    expect(normalizedHtml).toContain('north bank');
+    expect(normalizedHtml).toContain('full report');
+  });
+
+  it('exports markdown to EPUB as one-way smoke coverage', async () => {
+    const source = ROUNDTRIP_FIXTURES.story;
+    const epub = await markdownDocToEpub(source.doc, {
+      title: 'River Journal',
+      author: 'Squisq Test',
+    });
+
+    const zip = await JSZip.loadAsync(epub);
+    expect(zip.file('OEBPS/content.opf')).toBeTruthy();
+
+    const chapter = await zip.file('OEBPS/chapters/chapter-001.xhtml')?.async('string');
+    expect(chapter).toBeTruthy();
+
+    const normalized = (chapter ?? '').toLowerCase();
+    expect(normalized).toContain('river journal');
+    expect(normalized).toContain('north bank');
+  });
+
+  it('keeps high-value text through all bidirectional formats', async () => {
+    const source = ROUNDTRIP_FIXTURES.story;
+
+    const docxRoundTrip = await docxToMarkdownDoc(await markdownDocToDocx(source.doc));
+    const pptxRoundTrip = await pptxToMarkdownDoc(await markdownDocToPptx(source.doc));
+    const pdfRoundTrip = await pdfToMarkdownDoc(await markdownDocToPdf(source.doc));
+    const csvRoundTrip = await csvToMarkdownDoc(markdownDocToCsv(ROUNDTRIP_FIXTURES.table.doc));
+
+    const docxText = extractNormalizedText(docxRoundTrip);
+    const pptxText = extractNormalizedText(pptxRoundTrip);
+    const pdfText = extractNormalizedText(pdfRoundTrip);
+    const csvText = extractNormalizedText(csvRoundTrip);
+
+    expect(docxText).toContain('river journal');
+    expect(pptxText).toContain('river journal');
+    expect(pdfText).toContain('river journal');
+    expect(csvText).toContain('throughput');
+  });
+
+  it('asserts current XLSX export stub behavior', async () => {
+    await expect(markdownDocToXlsx(ROUNDTRIP_FIXTURES.table.doc)).rejects.toThrow(
+      'XLSX export is not yet implemented',
+    );
+  });
+});

package/src/__tests__/xlsxImport.test.ts

@@ -0,0 +1,80 @@
+/**
+ * Tests for XLSX import: xlsxToMarkdownDoc. Builds a minimal .xlsx fixture
+ * with the shared OOXML writer (dogfooding), then imports it.
+ */
+
+import { describe, expect, it } from 'vitest';
+import type { MarkdownHeading, MarkdownTable, MarkdownText } from '@bendyline/squisq/markdown';
+import { NS_R, NS_SML, REL_OFFICE_DOCUMENT } from '../ooxml/namespaces';
+import { createPackage } from '../ooxml/writer';
+import { xmlDeclaration } from '../ooxml/xmlUtils';
+import { xlsxToMarkdownDoc } from '../xlsx/import';
+
+const REL_WORKSHEET =
+  'http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet';
+const REL_SHARED_STRINGS =
+  'http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings';
+
+async function buildTestXlsx(): Promise<ArrayBuffer> {
+  const pkg = createPackage();
+
+  pkg.addPart(
+    'xl/workbook.xml',
+    `${xmlDeclaration()}<workbook xmlns="${NS_SML}" xmlns:r="${NS_R}">` +
+      `<sheets><sheet name="Data" sheetId="1" r:id="rId1"/></sheets></workbook>`,
+    'application/xml',
+  );
+  pkg.addPart(
+    'xl/sharedStrings.xml',
+    `${xmlDeclaration()}<sst xmlns="${NS_SML}">` +
+      `<si><t>Name</t></si><si><t>Age</t></si><si><t>Alice</t></si></sst>`,
+    'application/xml',
+  );
+  pkg.addPart(
+    'xl/worksheets/sheet1.xml',
+    `${xmlDeclaration()}<worksheet xmlns="${NS_SML}"><sheetData>` +
+      `<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>` +
+      `<row r="2"><c r="A2" t="s"><v>2</v></c><c r="B2"><v>30</v></c></row>` +
+      `</sheetData></worksheet>`,
+    'application/xml',
+  );
+
+  pkg.addRelationship('', { id: 'rId1', type: REL_OFFICE_DOCUMENT, target: 'xl/workbook.xml' });
+  pkg.addRelationship('xl/workbook.xml', {
+    id: 'rId1',
+    type: REL_WORKSHEET,
+    target: 'worksheets/sheet1.xml',
+  });
+  pkg.addRelationship('xl/workbook.xml', {
+    id: 'rId2',
+    type: REL_SHARED_STRINGS,
+    target: 'sharedStrings.xml',
+  });
+
+  return pkg.toArrayBuffer();
+}
+
+describe('xlsxToMarkdownDoc', () => {
+  it('imports a sheet as a heading + table, resolving shared strings', async () => {
+    const doc = await xlsxToMarkdownDoc(await buildTestXlsx());
+    expect(doc.type).toBe('document');
+
+    const heading = doc.children[0] as MarkdownHeading;
+    expect(heading.type).toBe('heading');
+    expect((heading.children[0] as MarkdownText).value).toBe('Data');
+
+    const table = doc.children[1] as MarkdownTable;
+    expect(table.type).toBe('table');
+    expect(table.children).toHaveLength(2);
+    const headerCell = table.children[0]!.children[0]!;
+    expect(headerCell.isHeader).toBe(true);
+    expect((headerCell.children[0] as MarkdownText).value).toBe('Name');
+    expect((table.children[1]!.children[0]!.children[0] as MarkdownText).value).toBe('Alice');
+    expect((table.children[1]!.children[1]!.children[0] as MarkdownText).value).toBe('30');
+  });
+
+  it('selects a single sheet by name without a heading', async () => {
+    const doc = await xlsxToMarkdownDoc(await buildTestXlsx(), { sheet: 'Data' });
+    expect(doc.children[0]!.type).toBe('table');
+  });
+});

package/src/csv/index.ts

@@ -0,0 +1,165 @@
+/**
+ * @bendyline/squisq-formats CSV Module
+ *
+ * Bridges CSV ↔ the squisq markdown table model. CSV isn't OOXML, so this
+ * module is self-contained (no jszip / DOMParser): a small RFC-4180 parser on
+ * the import side and a serializer on the export side. The first row is treated
+ * as the table header by default.
+ *
+ * @example
+ * ```ts
+ * import { csvToMarkdownDoc, markdownDocToCsv } from '@bendyline/squisq-formats/csv';
+ * ```
+ */
+
+import { markdownToDoc } from '@bendyline/squisq/doc';
+import type {
+  MarkdownDocument,
+  MarkdownTable,
+  MarkdownTableCell,
+  MarkdownTableRow,
+} from '@bendyline/squisq/markdown';
+import type { Doc } from '@bendyline/squisq/schemas';
+
+export interface CsvImportOptions {
+  /** Field delimiter. Default `,`. */
+  delimiter?: string;
+  /** Treat the first row as a header row. Default true. */
+  hasHeader?: boolean;
+}
+
+export interface CsvExportOptions {
+  /** Field delimiter. Default `,`. */
+  delimiter?: string;
+}
+
+async function toText(data: ArrayBuffer | Blob | string): Promise<string> {
+  if (typeof data === 'string') return data;
+  if (typeof Blob !== 'undefined' && data instanceof Blob) return data.text();
+  return new TextDecoder().decode(new Uint8Array(data as ArrayBuffer));
+}
+
+/** Parse CSV text into a grid of string cells (RFC 4180: quotes, escaped quotes). */
+export function parseCsv(text: string, delimiter = ','): string[][] {
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = '';
+  let inQuotes = false;
+  let sawAny = false;
+  const pushField = () => {
+    row.push(field);
+    field = '';
+  };
+  const pushRow = () => {
+    pushField();
+    rows.push(row);
+    row = [];
+  };
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]!;
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        field += ch;
+      }
+      continue;
+    }
+    if (ch === '"') {
+      inQuotes = true;
+      sawAny = true;
+      continue;
+    }
+    if (ch === delimiter) {
+      sawAny = true;
+      pushField();
+      continue;
+    }
+    if (ch === '\n' || ch === '\r') {
+      if (ch === '\r' && text[i + 1] === '\n') i++;
+      pushRow();
+      sawAny = false;
+      continue;
+    }
+    sawAny = true;
+    field += ch;
+  }
+  // Flush a trailing field/row only if the last line wasn't terminated.
+  if (field !== '' || row.length > 0 || sawAny) pushRow();
+  return rows;
+}
+
+function rowsToTable(rows: string[][], hasHeader: boolean): MarkdownTable {
+  const maxCols = rows.reduce((m, r) => Math.max(m, r.length), 1);
+  const mdRows: MarkdownTableRow[] = rows.map((cells, rowIdx) => {
+    const children: MarkdownTableCell[] = [];
+    for (let c = 0; c < maxCols; c++) {
+      const value = cells[c] ?? '';
+      children.push({
+        type: 'tableCell',
+        ...(hasHeader && rowIdx === 0 ? { isHeader: true } : {}),
+        children: value ? [{ type: 'text', value }] : [],
+      });
+    }
+    return { type: 'tableRow', children };
+  });
+  return { type: 'table', children: mdRows };
+}
+
+/** Convert CSV to a MarkdownDocument containing a single table. */
+export async function csvToMarkdownDoc(
+  data: ArrayBuffer | Blob | string,
+  options: CsvImportOptions = {},
+): Promise<MarkdownDocument> {
+  const text = await toText(data);
+  const rows = parseCsv(text, options.delimiter ?? ',');
+  const hasHeader = options.hasHeader ?? true;
+  const children = rows.length > 0 ? [rowsToTable(rows, hasHeader)] : [];
+  return { type: 'document', children };
+}
+
+/** Convert CSV to a squisq Doc. */
+export async function csvToDoc(
+  data: ArrayBuffer | Blob | string,
+  options: CsvImportOptions = {},
+): Promise<Doc> {
+  return markdownToDoc(await csvToMarkdownDoc(data, options));
+}
+
+function escapeCsvField(value: string, delimiter: string): string {
+  if (value.includes('"') || value.includes(delimiter) || /[\r\n]/.test(value)) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+  return value;
+}
+
+function cellText(cell: MarkdownTableCell): string {
+  // Flatten inline children to plain text (CSV has no formatting).
+  const walk = (nodes: unknown[]): string =>
+    nodes
+      .map((n) => {
+        const node = n as { type?: string; value?: string; children?: unknown[] };
+        if (node.value !== undefined) return node.value;
+        if (Array.isArray(node.children)) return walk(node.children);
+        return '';
+      })
+      .join('');
+  return walk(cell.children);
+}
+
+/** Serialize the first table in a MarkdownDocument to CSV text. */
+export function markdownDocToCsv(doc: MarkdownDocument, options: CsvExportOptions = {}): string {
+  const delimiter = options.delimiter ?? ',';
+  const table = doc.children.find((n) => n.type === 'table') as MarkdownTable | undefined;
+  if (!table) return '';
+  return table.children
+    .map((row) =>
+      row.children.map((cell) => escapeCsvField(cellText(cell), delimiter)).join(delimiter),
+    )
+    .join('\r\n');
+}