@markitdownjs/pack 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 MarkItDownJS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { pack, unpack } from "./pack.js";
+export type { PackBundle, PackManifest, PackOptions, PackCompression } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export { pack, unpack } from "./pack.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC"}

package/dist/pack.d.ts

@@ -0,0 +1,17 @@
+import type { ConversionResult } from "@markitdownjs/shared";
+import type { PackBundle, PackOptions } from "./types.js";
+/**
+ * Pack a ConversionResult into a portable bundle.
+ * The bundle contains compressed chunks and metadata that can be restored anywhere.
+ */
+export declare function pack(result: ConversionResult, options?: PackOptions): Promise<PackBundle>;
+/**
+ * Unpack a PackBundle back into a ConversionResult-like object.
+ */
+export declare function unpack(bundle: PackBundle): {
+    markdown: string;
+    chunks: any[];
+    metadata: Record<string, unknown>;
+    ast?: any;
+};
+//# sourceMappingURL=pack.d.ts.map

package/dist/pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.d.ts","sourceRoot":"","sources":["../src/pack.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,KAAK,EAAE,UAAU,EAAgB,WAAW,EAAE,MAAM,YAAY,CAAC;AAExE;;;GAGG;AACH,wBAAsB,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAoDnG;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,CAAC;CACX,CAWA"}

package/dist/pack.js

@@ -0,0 +1,60 @@
+/**
+ * Pack a ConversionResult into a portable bundle.
+ * The bundle contains compressed chunks and metadata that can be restored anywhere.
+ */
+export async function pack(result, options = {}) {
+    const { compression: _compression = "none", includeAst = false, includeChunks = true, metadata, } = options;
+    const chunks = result.chunks ?? [];
+    // Build the payload data.
+    const payloadData = {};
+    if (includeChunks) {
+        payloadData.chunks = chunks.map((c) => ({
+            id: c.id,
+            content: c.content,
+            metadata: c.metadata,
+            ast: includeAst ? c.ast : undefined,
+        }));
+    }
+    if (includeAst && result.ast) {
+        payloadData.ast = result.ast;
+    }
+    payloadData.markdown = result.markdown;
+    payloadData.metadata = result.metadata;
+    // Serialize to JSON.
+    const json = JSON.stringify(payloadData);
+    // Base64-encode (compression is reserved for future use).
+    const payload = btoa(unescape(encodeURIComponent(json)));
+    // Compute manifest.
+    const totalTokens = chunks.reduce((sum, c) => sum + (c.metadata.tokenCount ?? 0), 0);
+    const manifest = {
+        chunkCount: chunks.length,
+        tokenCount: totalTokens,
+    };
+    // Simple fingerprint from markdown content.
+    let hash = 0;
+    for (let i = 0; i < result.markdown.length; i++) {
+        hash = ((hash << 5) - hash + result.markdown.charCodeAt(i)) | 0;
+    }
+    manifest.fingerprint = `djb2:${Math.abs(hash).toString(16).padStart(8, "0")}`;
+    return {
+        format: "markitdownjs-pack-v1",
+        manifest,
+        payload,
+        metadata,
+    };
+}
+/**
+ * Unpack a PackBundle back into a ConversionResult-like object.
+ */
+export function unpack(bundle) {
+    // Decode base64 payload.
+    const json = decodeURIComponent(escape(atob(bundle.payload)));
+    const data = JSON.parse(json);
+    return {
+        markdown: data.markdown ?? "",
+        chunks: data.chunks ?? [],
+        metadata: data.metadata ?? {},
+        ast: data.ast,
+    };
+}
+//# sourceMappingURL=pack.js.map

package/dist/pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.js","sourceRoot":"","sources":["../src/pack.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,MAAwB,EAAE,UAAuB,EAAE;IAC5E,MAAM,EACJ,WAAW,EAAE,YAAY,GAAG,MAAM,EAClC,UAAU,GAAG,KAAK,EAClB,aAAa,GAAG,IAAI,EACpB,QAAQ,GACT,GAAG,OAAO,CAAC;IAEZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAEnC,0BAA0B;IAC1B,MAAM,WAAW,GAA4B,EAAE,CAAC;IAChD,IAAI,aAAa,EAAE,CAAC;QAClB,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;IACD,IAAI,UAAU,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;QAC7B,WAAW,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAC/B,CAAC;IACD,WAAW,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACvC,WAAW,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAEvC,qBAAqB;IACrB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAEzC,0DAA0D;IAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEzD,oBAAoB;IACpB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAiB;QAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,UAAU,EAAE,WAAW;KACxB,CAAC;IAEF,4CAA4C;IAC5C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;IACD,QAAQ,CAAC,WAAW,GAAG,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAE9E,OAAO;QACL,MAAM,EAAE,sBAAsB;QAC9B,QAAQ;QACR,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,MAAkB;IAMvC,yBAAyB;IACzB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE9B,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;QAC7B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;QAC7B,GAAG,EAAE,IAAI,CAAC,GAAG;KACd,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,36 @@
+/** Compression type for the pack payload */
+export type PackCompression = "none" | "gzip" | "brotli";
+/** Manifest for a packed bundle */
+export interface PackManifest {
+    /** Total number of chunks */
+    chunkCount: number;
+    /** Total token count across all chunks */
+    tokenCount: number;
+    /** Token reduction percentage vs raw text */
+    reduction?: string;
+    /** Content fingerprint for dedup */
+    fingerprint?: string;
+}
+/** Configuration for packing */
+export interface PackOptions {
+    /** Compression algorithm (default: "none") */
+    compression?: PackCompression;
+    /** Include the full AST in the bundle (default: false for smaller payload) */
+    includeAst?: boolean;
+    /** Include chunks in the bundle (default: true) */
+    includeChunks?: boolean;
+    /** Optional metadata to embed in the manifest */
+    metadata?: Record<string, unknown>;
+}
+/** A portable bundle produced by pack() */
+export interface PackBundle {
+    /** Format identifier */
+    format: "markitdownjs-pack-v1";
+    /** Manifest with summary stats */
+    manifest: PackManifest;
+    /** Base64-encoded compressed payload */
+    payload: string;
+    /** Embedded metadata */
+    metadata?: Record<string, unknown>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEzD,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,WAAW;IAC1B,8CAA8C;IAC9C,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,8EAA8E;IAC9E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mDAAmD;IACnD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iDAAiD;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,wBAAwB;IACxB,MAAM,EAAE,sBAAsB,CAAC;IAC/B,kCAAkC;IAClC,QAAQ,EAAE,YAAY,CAAC;IACvB,wCAAwC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@markitdownjs/pack",
+  "version": "0.1.0",
+  "description": "Standardized portable output format for MarkItDownJS",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@markitdownjs/shared": "0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.5.0"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "dev": "tsc --watch --project tsconfig.json",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist tsconfig.tsbuildinfo"
+  }
+}