mhtml-stream 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Erik Brinkman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,27 @@
+MHTML Stream
+============
+[![build](https://github.com/erikbrinkman/mhtml-stream/actions/workflows/node.js.yml/badge.svg)](https://github.com/erikbrinkman/mhtml-stream/actions/workflows/node.js.yml)
+[![docs](https://img.shields.io/badge/docs-docs-blue)](https://erikbrinkman.github.io/mhtml-stream/)
+
+Zero-dependency library for parsing MHTML data as streams using modern WHATWG
+streams and async iterators. Because it relies on modern cross javascript
+standards it works out-of-the-box in all javascript environments, with only a
+little tweaking necessary for module definitions.
+
+Usage
+-----
+
+```javascript
+import { parseMhtml } from "mhtml-stream";
+
+for await (const { headers, content } of parseMhtml(...)) {
+  // headers : a key-value object with the header information
+
+  // content : a Uint8Array of the raw data, if you want as a string, `new
+  //   TextDecoder().decode(content)` should work if the contents were utf-8 /
+  //   ascii encoded
+
+  // NOTE in many MHTML files, the initial file is empty and contains headers
+  // for how to parse each individual included file.
+}
+```

package/dist/base64.d.ts

@@ -0,0 +1,8 @@
+/**
+ * decode base 64 string as bytes
+ */
+export declare function decodeBase64(sBase64: string, nBlocksSize?: number): Uint8Array;
+/**
+ * encode bytes as base64 string
+ */
+export declare function encodeBase64(aBytes: Uint8Array): string;

package/dist/base64.js

@@ -0,0 +1,77 @@
+/*\
+|*|
+|*|  Base64 / binary data / UTF-8 strings utilities
+|*|
+|*|  https://developer.mozilla.org/en-US/docs/Web/JavaScript/Base64_encoding_and_decoding
+|*|
+\*/
+/* Array of bytes to Base64 string decoding */
+function b64ToUint6(nChr) {
+    return nChr > 64 && nChr < 91
+        ? nChr - 65
+        : nChr > 96 && nChr < 123
+            ? nChr - 71
+            : nChr > 47 && nChr < 58
+                ? nChr + 4
+                : nChr === 43
+                    ? 62
+                    : nChr === 47
+                        ? 63
+                        : /* istanbul ignore next */
+                            0;
+}
+/**
+ * decode base 64 string as bytes
+ */
+export function decodeBase64(sBase64, nBlocksSize) {
+    const sB64Enc = sBase64.replace(/[^A-Za-z0-9+/]/g, "");
+    const nInLen = sB64Enc.length;
+    const nOutLen = nBlocksSize
+        ? Math.ceil(((nInLen * 3 + 1) >> 2) / nBlocksSize) * nBlocksSize
+        : (nInLen * 3 + 1) >> 2;
+    const taBytes = new Uint8Array(nOutLen);
+    for (let nUint24 = 0, nOutIdx = 0, nInIdx = 0; nInIdx < nInLen; nInIdx++) {
+        const nMod4 = nInIdx & 3;
+        nUint24 |= b64ToUint6(sB64Enc.charCodeAt(nInIdx)) << (6 * (3 - nMod4));
+        if (nMod4 === 3 || nInLen - nInIdx === 1) {
+            for (let nMod3 = 0; nMod3 < 3 && nOutIdx < nOutLen; nMod3++, nOutIdx++) {
+                taBytes[nOutIdx] = (nUint24 >>> ((16 >>> nMod3) & 24)) & 255;
+            }
+            nUint24 = 0;
+        }
+    }
+    return taBytes;
+}
+/* Base64 string to array encoding */
+function uint6ToB64(nUint6) {
+    return nUint6 < 26
+        ? nUint6 + 65
+        : nUint6 < 52
+            ? nUint6 + 71
+            : nUint6 < 62
+                ? nUint6 - 4
+                : nUint6 === 62
+                    ? 43
+                    : nUint6 === 63
+                        ? 47
+                        : /* istanbul ignore next */
+                            65;
+}
+/**
+ * encode bytes as base64 string
+ */
+export function encodeBase64(aBytes) {
+    const nLen = aBytes.length;
+    let nMod3 = 2;
+    let sB64Enc = "";
+    for (let nUint24 = 0, nIdx = 0; nIdx < nLen; nIdx++) {
+        nMod3 = nIdx % 3;
+        nUint24 |= aBytes[nIdx] << ((16 >>> nMod3) & 24);
+        if (nMod3 === 2 || aBytes.length - nIdx === 1) {
+            sB64Enc += String.fromCodePoint(uint6ToB64((nUint24 >>> 18) & 63), uint6ToB64((nUint24 >>> 12) & 63), uint6ToB64((nUint24 >>> 6) & 63), uint6ToB64(nUint24 & 63));
+            nUint24 = 0;
+        }
+    }
+    return (sB64Enc.substr(0, sB64Enc.length - 2 + nMod3) +
+        (nMod3 === 2 ? "" : nMod3 === 1 ? "=" : "=="));
+}

package/dist/index.cjs.min.js

@@ -0,0 +1 @@
+var y=Object.defineProperty;var S=Object.getOwnPropertyDescriptor;var $=Object.getOwnPropertyNames;var k=Object.prototype.hasOwnProperty;var q=e=>y(e,"__esModule",{value:!0});var D=(e,t)=>{for(var r in t)y(e,r,{get:t[r],enumerable:!0})},L=(e,t,r,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of $(t))!k.call(e,o)&&(r||o!=="default")&&y(e,o,{get:()=>t[o],enumerable:!(n=S(t,o))||n.enumerable});return e};var P=(e=>(t,r)=>e&&e.get(t)||(r=L(q({}),t,1),e&&e.set(t,r),r))(typeof WeakMap!="undefined"?new WeakMap:0);var K={};D(K,{MhtmlHeaders:()=>h,parseMhtml:()=>G});function Q(e){return e>64&&e<91?e-65:e>96&&e<123?e-71:e>47&&e<58?e+4:e===43?62:e===47?63:0}function f(e,t){let r=e.replace(/[^A-Za-z0-9+/]/g,""),n=r.length,o=t?Math.ceil((n*3+1>>2)/t)*t:n*3+1>>2,s=new Uint8Array(o);for(let a=0,i=0,d=0;d<n;d++){let l=d&3;if(a|=Q(r.charCodeAt(d))<<6*(3-l),l===3||n-d===1){for(let c=0;c<3&&i<o;c++,i++)s[i]=a>>>(16>>>c&24)&255;a=0}}return s}function j(e){return ArrayBuffer.isView(e)?new Uint8Array(e.buffer,e.byteOffset,e.byteLength):new Uint8Array(e)}function g(e,t){if(e.length!==t.length)return!1;if(e.byteOffset%4===t.byteOffset%4){let r=(4-e.byteOffset%4)%4,n=Math.floor((e.byteLength-r)/4),o=r+n*4,s=new Uint32Array(e.buffer,e.byteOffset+r,n),a=new Uint32Array(t.buffer,t.byteOffset+r,n);for(let i=0;i<r;++i)if(e[i]!==t[i])return!1;for(let i=0;i<n;++i)if(s[i]!==a[i])return!1;for(let i=o;i<e.length;++i)if(e[i]!==t[i])return!1;return!0}else{for(let r of e.keys())if(e[r]!==t[r])return!1;return!0}}function A(e){return{[Symbol.asyncIterator](){let t=e.getReader();return{async next(){let r=await t.read();return r.done?{done:!0,value:void 0}:{done:!1,value:r.value}}}}}}function H(e,t){return e.findIndex((r,n)=>{if(r!==t[0]||n+t.length>e.length)return!1;for(let o=1;o<t.length;++o)if(e[n+o]!==t[o])return!1;return!0})}async function*p(e,t){let r=new Uint8Array(0);for await(let n of e){let o=j(n);r=r.length?I([r,o]):o;let s;for(;(s=H(r,t))!==-1;)yield r.subarray(0,s),r=r.subarray(s+t.length)}yield r}function I(e){let t=e.reduce((o,s)=>o+s.length,0),r=new Uint8Array(t),n=0;for(let o of e)r.set(o,n),n+=o.length;return r}async function m(e){let t=[];for await(let r of e)t.push(r);return I(t)}var R=new Uint8Array([10]);async function*U(e,t=R){for await(let r of e){let n=new Uint8Array(r.length+t.length),o=!1,s=0;for(let a=0;a<r.length;++a){let i=r[a];if(i>=128)throw new Error(`got non-ascii character when decoding quoted printable: ${r}`);if(i!==61)n[s++]=i;else{let d=r[++a];if(d===void 0)o=!0;else{let l=r[++a];if(l===void 0)throw new Error("quoted printable escape (=) was not followed by two bytes");let c=parseInt(String.fromCharCode(d),16);c*=16,c+=parseInt(String.fromCharCode(l),16),n[s++]=c}}}o||(n.set(t,s),s+=t.length),yield n.subarray(0,s)}}var W=new TextDecoder;async function*x(e){for await(let t of e)yield f(W.decode(t))}async function*b(e){for await(let t of e)yield t}function v(){throw new Error("binary transfer-encoding is explicitly not supported and trying to add an implementation will likely result in unexpected results, but if you want to ignore anyway, set binary to `decode8bit`")}var E=new TextEncoder,M=new TextDecoder,h=class{#e=new Map;[Symbol.iterator](){return this.entries()}append(t,r){let n=this.#e.get(t);n===void 0?this.#e.set(t,[r]):n.push(r)}*entries(t=", "){for(let[r,n]of this.#e.entries())yield[r,n.join(t)]}*entriesAll(){for(let[t,r]of this.#e.entries())for(let n of r)yield[t,n]}get(t,r=", "){let n=this.#e.get(t);return n===void 0?null:n.join(r)}getAll(t){return this.#e.get(t)??[]}has(t){return this.#e.has(t)}keys(){return this.#e.keys()}*values(t=", "){for(let r of this.#e.values())yield r.join(t)}*valuesAll(){for(let t of this.#e.values())yield*t}};function F(e){let t=new Uint8Array(e.length),r=0;for(let n=0;n<e.length;++n){let o=e.charCodeAt(n),s;if(o>=128)throw new Error(`got non-ascii character when decoding q-quoted word: "${e}"`);o===95?s=32:o===61?(s=parseInt(e[++n],16),s*=16,s+=parseInt(e[++n],16)):s=o,t[r++]=s}return t.subarray(0,r)}var N=/^=\?([^?\s]+)\?([BQ])\?([^?\s]+)\?=$/;function B(e){let t=N.exec(e);if(t){let[,r,n,o]=t,s;if(n==="Q")s=F(o);else if(n==="B")s=f(o);else throw new Error(`unknown encoding for header value encoding: ${n}`);return new TextDecoder(r).decode(s)}else return e}async function J(e){let t=new h,r="",n="";for(;;){let{done:o,value:s}=await e.next();if(o)throw new Error("didn't find an empty line to signify the end of header parsing");let a=M.decode(s);if(/^\s/.test(a))n+=" ",n+=B(a.trimStart());else if(r&&t.append(r,n),a){let i=a.indexOf(": ");if(i===-1)throw new Error(`header line didn't have key-value delimiter: "${a}"`);r=a.slice(0,i),n=B(a.slice(i+2))}else return t}}var V=new Map([["7bit",b],["base64",x],["quoted-printable",U],["8bit",b],["binary",v]]);function Z(e){let t=e.get("Content-Type");if(t===null)throw new Error(`first headers didn't contain a content type: ${JSON.stringify(Object.fromEntries(e))}`);let r,n=!1;for(let a of t.split("; "))a.startsWith("multipart/")?n=!0:a.startsWith("boundary=")&&(r=a.slice(9),r.startsWith('"')&&r.endsWith('"')&&(r=r.slice(1,-1)));if(!n||r===void 0)throw new Error("first content type header didn't contain 'multipart/...' and a boundary string");let o=E.encode(`--${r}`),s=E.encode(`--${r}--`);return[o,s]}async function*G(e,{decoderOverrides:t=new Map}={}){let r=new Map([...V.entries(),...t.entries()]),n=new Uint8Array([13,10]),o=p(A(e),n),s=null,a=!0;for(;a;){let i=await J(o),[d,l]=s??(s=Z(i)),c=i.get("Content-Transfer-Encoding")??"7bit",w=r.get(c);if(w===void 0)throw new Error(`unhandled encoding type: ${c}`);let O=await m(w({[Symbol.asyncIterator](){return{async next(){let{done:T,value:u}=await o.next();if(T)throw new Error(`stream didn't end with the appropriate termination boundary: ${M.decode(l)}`);return g(u,d)?{done:!0,value:void 0}:g(u,l)?(a=!1,{done:!0,value:void 0}):{value:u}}}}}));yield{headers:i,content:O}}}module.exports=P(K);

package/dist/index.d.ts

@@ -0,0 +1,81 @@
+/**
+ * class for storing headers parse from MHTML
+ *
+ * Tries to somewhat mimic the behavior of the fetch-api Headers object, with
+ * some differences, notably that it does no validation.
+ */
+export declare class MhtmlHeaders {
+    #private;
+    [Symbol.iterator](): Iterator<[string, string]>;
+    /**
+     * add a key-value pair
+     *
+     * If key is already present it will be appended.
+     */
+    append(key: string, value: string): void;
+    /**
+     * iterate over all key-value pairs
+     *
+     * Multiple values for the same key will be joined in the order they were
+     * added by `delim`.
+     */
+    entries(delim?: string): IterableIterator<[string, string]>;
+    /**
+     * iterate over all key-value pairs
+     *
+     * Multiple values for the same key will get iterated in the order they were
+     * added.
+     */
+    entriesAll(): IterableIterator<[string, string]>;
+    /**
+     * get the value for a key
+     *
+     * If the key is missing, null will be returned, if multiple values for the
+     * key are present, they will be joined be `delim`.
+     */
+    get(key: string, delim?: string): string | null;
+    /**
+     * get all values for a key
+     */
+    getAll(key: string): string[];
+    /**
+     * whether key has any values associated with it
+     */
+    has(key: string): boolean;
+    /**
+     * all keys with at least one value
+     */
+    keys(): Iterable<string>;
+    /**
+     * iterate over all values
+     *
+     * If a key has multiple values they will be joined by `delim`.
+     */
+    values(delim?: string): IterableIterator<string>;
+    /**
+     * iterate over all values added
+     */
+    valuesAll(): IterableIterator<string>;
+}
+/**
+ * a file that was encoded in MHTML format
+ */
+export interface MhtmlFile {
+    headers: MhtmlHeaders;
+    content: Uint8Array;
+}
+/**
+ * The decoder of any Content-Transfer-Encoding
+ */
+export interface Decoder {
+    (lines: AsyncIterable<Uint8Array>): AsyncIterable<Uint8Array>;
+}
+/**
+ * parse a readable stream into an async iterator of MHTML files
+ *
+ * decoderOverrides can be used to overwrite default encoders or specify your
+ * own if a Content-Transfer-Encoding isn't handled properly
+ */
+export declare function parseMhtml(stream: ReadableStream<ArrayBuffer>, { decoderOverrides, }?: {
+    decoderOverrides?: Map<string, Decoder>;
+}): AsyncIterableIterator<MhtmlFile>;

package/dist/index.iife.min.js

@@ -0,0 +1 @@
+var mhtmlStream=(()=>{var y=Object.defineProperty;var S=Object.getOwnPropertyDescriptor;var $=Object.getOwnPropertyNames;var k=Object.prototype.hasOwnProperty;var q=e=>y(e,"__esModule",{value:!0});var D=(e,t)=>{for(var r in t)y(e,r,{get:t[r],enumerable:!0})},L=(e,t,r,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of $(t))!k.call(e,o)&&(r||o!=="default")&&y(e,o,{get:()=>t[o],enumerable:!(n=S(t,o))||n.enumerable});return e};var P=(e=>(t,r)=>e&&e.get(t)||(r=L(q({}),t,1),e&&e.set(t,r),r))(typeof WeakMap!="undefined"?new WeakMap:0);var K={};D(K,{MhtmlHeaders:()=>h,parseMhtml:()=>G});function Q(e){return e>64&&e<91?e-65:e>96&&e<123?e-71:e>47&&e<58?e+4:e===43?62:e===47?63:0}function f(e,t){let r=e.replace(/[^A-Za-z0-9+/]/g,""),n=r.length,o=t?Math.ceil((n*3+1>>2)/t)*t:n*3+1>>2,s=new Uint8Array(o);for(let a=0,i=0,d=0;d<n;d++){let l=d&3;if(a|=Q(r.charCodeAt(d))<<6*(3-l),l===3||n-d===1){for(let c=0;c<3&&i<o;c++,i++)s[i]=a>>>(16>>>c&24)&255;a=0}}return s}function j(e){return ArrayBuffer.isView(e)?new Uint8Array(e.buffer,e.byteOffset,e.byteLength):new Uint8Array(e)}function g(e,t){if(e.length!==t.length)return!1;if(e.byteOffset%4===t.byteOffset%4){let r=(4-e.byteOffset%4)%4,n=Math.floor((e.byteLength-r)/4),o=r+n*4,s=new Uint32Array(e.buffer,e.byteOffset+r,n),a=new Uint32Array(t.buffer,t.byteOffset+r,n);for(let i=0;i<r;++i)if(e[i]!==t[i])return!1;for(let i=0;i<n;++i)if(s[i]!==a[i])return!1;for(let i=o;i<e.length;++i)if(e[i]!==t[i])return!1;return!0}else{for(let r of e.keys())if(e[r]!==t[r])return!1;return!0}}function A(e){return{[Symbol.asyncIterator](){let t=e.getReader();return{async next(){let r=await t.read();return r.done?{done:!0,value:void 0}:{done:!1,value:r.value}}}}}}function H(e,t){return e.findIndex((r,n)=>{if(r!==t[0]||n+t.length>e.length)return!1;for(let o=1;o<t.length;++o)if(e[n+o]!==t[o])return!1;return!0})}async function*p(e,t){let r=new Uint8Array(0);for await(let n of e){let o=j(n);r=r.length?I([r,o]):o;let s;for(;(s=H(r,t))!==-1;)yield r.subarray(0,s),r=r.subarray(s+t.length)}yield r}function I(e){let t=e.reduce((o,s)=>o+s.length,0),r=new Uint8Array(t),n=0;for(let o of e)r.set(o,n),n+=o.length;return r}async function m(e){let t=[];for await(let r of e)t.push(r);return I(t)}var R=new Uint8Array([10]);async function*U(e,t=R){for await(let r of e){let n=new Uint8Array(r.length+t.length),o=!1,s=0;for(let a=0;a<r.length;++a){let i=r[a];if(i>=128)throw new Error(`got non-ascii character when decoding quoted printable: ${r}`);if(i!==61)n[s++]=i;else{let d=r[++a];if(d===void 0)o=!0;else{let l=r[++a];if(l===void 0)throw new Error("quoted printable escape (=) was not followed by two bytes");let c=parseInt(String.fromCharCode(d),16);c*=16,c+=parseInt(String.fromCharCode(l),16),n[s++]=c}}}o||(n.set(t,s),s+=t.length),yield n.subarray(0,s)}}var W=new TextDecoder;async function*x(e){for await(let t of e)yield f(W.decode(t))}async function*b(e){for await(let t of e)yield t}function v(){throw new Error("binary transfer-encoding is explicitly not supported and trying to add an implementation will likely result in unexpected results, but if you want to ignore anyway, set binary to `decode8bit`")}var E=new TextEncoder,M=new TextDecoder,h=class{#e=new Map;[Symbol.iterator](){return this.entries()}append(t,r){let n=this.#e.get(t);n===void 0?this.#e.set(t,[r]):n.push(r)}*entries(t=", "){for(let[r,n]of this.#e.entries())yield[r,n.join(t)]}*entriesAll(){for(let[t,r]of this.#e.entries())for(let n of r)yield[t,n]}get(t,r=", "){let n=this.#e.get(t);return n===void 0?null:n.join(r)}getAll(t){return this.#e.get(t)??[]}has(t){return this.#e.has(t)}keys(){return this.#e.keys()}*values(t=", "){for(let r of this.#e.values())yield r.join(t)}*valuesAll(){for(let t of this.#e.values())yield*t}};function F(e){let t=new Uint8Array(e.length),r=0;for(let n=0;n<e.length;++n){let o=e.charCodeAt(n),s;if(o>=128)throw new Error(`got non-ascii character when decoding q-quoted word: "${e}"`);o===95?s=32:o===61?(s=parseInt(e[++n],16),s*=16,s+=parseInt(e[++n],16)):s=o,t[r++]=s}return t.subarray(0,r)}var N=/^=\?([^?\s]+)\?([BQ])\?([^?\s]+)\?=$/;function B(e){let t=N.exec(e);if(t){let[,r,n,o]=t,s;if(n==="Q")s=F(o);else if(n==="B")s=f(o);else throw new Error(`unknown encoding for header value encoding: ${n}`);return new TextDecoder(r).decode(s)}else return e}async function J(e){let t=new h,r="",n="";for(;;){let{done:o,value:s}=await e.next();if(o)throw new Error("didn't find an empty line to signify the end of header parsing");let a=M.decode(s);if(/^\s/.test(a))n+=" ",n+=B(a.trimStart());else if(r&&t.append(r,n),a){let i=a.indexOf(": ");if(i===-1)throw new Error(`header line didn't have key-value delimiter: "${a}"`);r=a.slice(0,i),n=B(a.slice(i+2))}else return t}}var V=new Map([["7bit",b],["base64",x],["quoted-printable",U],["8bit",b],["binary",v]]);function Z(e){let t=e.get("Content-Type");if(t===null)throw new Error(`first headers didn't contain a content type: ${JSON.stringify(Object.fromEntries(e))}`);let r,n=!1;for(let a of t.split("; "))a.startsWith("multipart/")?n=!0:a.startsWith("boundary=")&&(r=a.slice(9),r.startsWith('"')&&r.endsWith('"')&&(r=r.slice(1,-1)));if(!n||r===void 0)throw new Error("first content type header didn't contain 'multipart/...' and a boundary string");let o=E.encode(`--${r}`),s=E.encode(`--${r}--`);return[o,s]}async function*G(e,{decoderOverrides:t=new Map}={}){let r=new Map([...V.entries(),...t.entries()]),n=new Uint8Array([13,10]),o=p(A(e),n),s=null,a=!0;for(;a;){let i=await J(o),[d,l]=s??(s=Z(i)),c=i.get("Content-Transfer-Encoding")??"7bit",w=r.get(c);if(w===void 0)throw new Error(`unhandled encoding type: ${c}`);let O=await m(w({[Symbol.asyncIterator](){return{async next(){let{done:T,value:u}=await o.next();if(T)throw new Error(`stream didn't end with the appropriate termination boundary: ${M.decode(l)}`);return g(u,d)?{done:!0,value:void 0}:g(u,l)?(a=!1,{done:!0,value:void 0}):{value:u}}}}}));yield{headers:i,content:O}}}return P(K);})();

package/dist/index.js

@@ -0,0 +1,295 @@
+import { decodeBase64 as decodeBEncoding } from "./base64";
+import { asIterable, bytesEqual, collect, decodeBase64, decodeBinary, decodeIdentity, decodeQuotedPrintable, splitStream, } from "./utils";
+// NOTE we use this for ascii because it's faster than manual, but may cause
+// unexpected errors if the assumption is violated
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+/**
+ * class for storing headers parse from MHTML
+ *
+ * Tries to somewhat mimic the behavior of the fetch-api Headers object, with
+ * some differences, notably that it does no validation.
+ */
+export class MhtmlHeaders {
+    #raw = new Map();
+    [Symbol.iterator]() {
+        return this.entries();
+    }
+    /**
+     * add a key-value pair
+     *
+     * If key is already present it will be appended.
+     */
+    append(key, value) {
+        const list = this.#raw.get(key);
+        if (list === undefined) {
+            this.#raw.set(key, [value]);
+        }
+        else {
+            list.push(value);
+        }
+    }
+    /**
+     * iterate over all key-value pairs
+     *
+     * Multiple values for the same key will be joined in the order they were
+     * added by `delim`.
+     */
+    *entries(delim = ", ") {
+        for (const [key, values] of this.#raw.entries()) {
+            yield [key, values.join(delim)];
+        }
+    }
+    /**
+     * iterate over all key-value pairs
+     *
+     * Multiple values for the same key will get iterated in the order they were
+     * added.
+     */
+    *entriesAll() {
+        for (const [key, values] of this.#raw.entries()) {
+            for (const val of values) {
+                yield [key, val];
+            }
+        }
+    }
+    /**
+     * get the value for a key
+     *
+     * If the key is missing, null will be returned, if multiple values for the
+     * key are present, they will be joined be `delim`.
+     */
+    get(key, delim = ", ") {
+        const vals = this.#raw.get(key);
+        if (vals === undefined) {
+            return null;
+        }
+        else {
+            return vals.join(delim);
+        }
+    }
+    /**
+     * get all values for a key
+     */
+    getAll(key) {
+        return this.#raw.get(key) ?? [];
+    }
+    /**
+     * whether key has any values associated with it
+     */
+    has(key) {
+        return this.#raw.has(key);
+    }
+    /**
+     * all keys with at least one value
+     */
+    keys() {
+        return this.#raw.keys();
+    }
+    /**
+     * iterate over all values
+     *
+     * If a key has multiple values they will be joined by `delim`.
+     */
+    *values(delim = ", ") {
+        for (const vals of this.#raw.values()) {
+            yield vals.join(delim);
+        }
+    }
+    /**
+     * iterate over all values added
+     */
+    *valuesAll() {
+        for (const vals of this.#raw.values()) {
+            yield* vals;
+        }
+    }
+}
+/**
+ * decode a q-encoded token
+ */
+function decodeQEncoding(text) {
+    const res = new Uint8Array(text.length);
+    let destInd = 0;
+    for (let ind = 0; ind < text.length; ++ind) {
+        const code = text.charCodeAt(ind);
+        let val;
+        if (code >= 128) {
+            throw new Error(`got non-ascii character when decoding q-quoted word: "${text}"`);
+        }
+        else if (code === 95) {
+            // Q encoding replaces underscore with space
+            val = 32;
+        }
+        else if (code === 61) {
+            // encoded character
+            val = parseInt(text[++ind], 16);
+            val *= 16;
+            val += parseInt(text[++ind], 16);
+        }
+        else {
+            // residual ascii
+            val = code;
+        }
+        res[destInd++] = val;
+    }
+    return res.subarray(0, destInd);
+}
+const encodeFormat = /^=\?([^?\s]+)\?([BQ])\?([^?\s]+)\?=$/;
+/**
+ * decode a header line that may have an extra encoding in it
+ */
+function decodeLine(line) {
+    // TODO this might make more sense as a function as part of a regex replaceAll
+    const match = encodeFormat.exec(line);
+    if (match) {
+        const [, charset, encoding, text] = match;
+        let buff;
+        /* istanbul ignore else */
+        if (encoding === "Q") {
+            buff = decodeQEncoding(text);
+        }
+        else if (encoding === "B") {
+            buff = decodeBEncoding(text);
+        }
+        else {
+            // NOTE should be impossible
+            throw new Error(`unknown encoding for header value encoding: ${encoding}`);
+        }
+        return new TextDecoder(charset).decode(buff);
+    }
+    else {
+        return line;
+    }
+}
+/**
+ * parse headers from line delimited buffers
+ *
+ * @remarks
+ * This is not fully compatible header parsing, since we overwrite identical
+ * keys instead of storing multiple
+ */
+async function parseHeaders(iter) {
+    const headers = new MhtmlHeaders();
+    let key = "";
+    let val = "";
+    for (;;) {
+        const { done, value } = await iter.next();
+        if (done) {
+            throw new Error("didn't find an empty line to signify the end of header parsing");
+        }
+        const line = decoder.decode(value);
+        if (/^\s/.test(line)) {
+            // header folded
+            val += " ";
+            val += decodeLine(line.trimStart());
+        }
+        else {
+            if (key) {
+                headers.append(key, val);
+            }
+            if (line) {
+                const delim = line.indexOf(": ");
+                if (delim === -1) {
+                    throw new Error(`header line didn't have key-value delimiter: "${line}"`);
+                }
+                key = line.slice(0, delim);
+                val = decodeLine(line.slice(delim + 2));
+            }
+            else {
+                return headers;
+            }
+        }
+    }
+}
+// Default decoders
+const defaultDecoders = new Map([
+    ["7bit", decodeIdentity],
+    ["base64", decodeBase64],
+    ["quoted-printable", decodeQuotedPrintable],
+    ["8bit", decodeIdentity],
+    ["binary", decodeBinary],
+]);
+/**
+ * extract the boundary condition from a multipart header
+ *
+ * returns the boundary and terminating condition as Uint8Arrays
+ */
+function getBoundary(headers) {
+    const contentType = headers.get("Content-Type");
+    if (contentType === null) {
+        throw new Error(`first headers didn't contain a content type: ${JSON.stringify(Object.fromEntries(headers))}`);
+    }
+    let bound = undefined;
+    let multipart = false;
+    for (const field of contentType.split("; ")) {
+        if (field.startsWith("multipart/")) {
+            multipart = true;
+        }
+        else if (field.startsWith("boundary=")) {
+            bound = field.slice(9);
+            // TODO handling of quoted fields is not great
+            if (bound.startsWith('"') && bound.endsWith('"')) {
+                bound = bound.slice(1, -1);
+            }
+        }
+    }
+    if (!multipart || bound === undefined) {
+        throw new Error(`first content type header didn't contain 'multipart/...' and a boundary string`);
+    }
+    const boundary = encoder.encode(`--${bound}`);
+    const terminus = encoder.encode(`--${bound}--`);
+    return [boundary, terminus];
+}
+/**
+ * parse a readable stream into an async iterator of MHTML files
+ *
+ * decoderOverrides can be used to overwrite default encoders or specify your
+ * own if a Content-Transfer-Encoding isn't handled properly
+ */
+export async function* parseMhtml(stream, { decoderOverrides = new Map(), } = {}) {
+    // initial setup
+    const decoders = new Map([
+        ...defaultDecoders.entries(),
+        ...decoderOverrides.entries(),
+    ]);
+    const crlf = new Uint8Array([13, 10]);
+    const lines = splitStream(asIterable(stream), crlf);
+    let bound = null;
+    let cont = true;
+    while (cont) {
+        // parse out headers and get encoding for content
+        const headers = await parseHeaders(lines);
+        const [boundary, terminus] = bound ?? (bound = getBoundary(headers));
+        const encoding = headers.get("Content-Transfer-Encoding") ?? "7bit";
+        const decode = decoders.get(encoding);
+        if (decode === undefined) {
+            throw new Error(`unhandled encoding type: ${encoding}`);
+        }
+        // create line iterator that only iterates over content lines (checking for
+        // the boundaries), then pass into decoder
+        const content = await collect(decode({
+            [Symbol.asyncIterator]() {
+                return {
+                    async next() {
+                        const { done, value } = await lines.next();
+                        if (done) {
+                            throw new Error(`stream didn't end with the appropriate termination boundary: ${decoder.decode(terminus)}`);
+                        }
+                        else if (bytesEqual(value, boundary)) {
+                            return { done: true, value: undefined };
+                        }
+                        else if (bytesEqual(value, terminus)) {
+                            cont = false;
+                            return { done: true, value: undefined };
+                        }
+                        else {
+                            return { value };
+                        }
+                    },
+                };
+            },
+        }));
+        yield { headers, content };
+    }
+}

package/dist/utils.d.ts

@@ -0,0 +1,53 @@
+/**
+ * create a Uint8Array from a buffer
+ *
+ * This method is necessary since creating a Uint8Array from a view will copy
+ * the elements instead of creating a view of the underling buffer.
+ */
+export declare function toBytes(buff: ArrayBuffer): Uint8Array;
+/**
+ * Compare two byte arrays for equality
+ */
+export declare function bytesEqual(left: Uint8Array, right: Uint8Array): boolean;
+/**
+ * convert readable stream to async iterator
+ */
+export declare function asIterable<T extends ArrayBuffer>(stream: ReadableStream<T>): AsyncIterable<T>;
+/**
+ * Find index of one byte array in another
+ */
+export declare function indexOf(haystack: Uint8Array, needle: Uint8Array): number;
+/**
+ * Split a stream of bytes
+ *
+ * Takes a stream of data modeled as an async iterator of ArrayBuffer for
+ * compatibility between node and web, and splits it into an async iterator
+ * where each value is delimited by the split sequence.
+ */
+export declare function splitStream(iter: AsyncIterable<ArrayBuffer>, split: Uint8Array): AsyncIterableIterator<Uint8Array>;
+/**
+ * collect an async iterable of buffers into one
+ */
+export declare function collect(stream: AsyncIterable<Uint8Array>): Promise<Uint8Array>;
+/**
+ * decoder for quoted printable
+ *
+ * If quoted printable "lines" aren't escaped with an "=" then a new line needs
+ * to be inserted. We use `newLine` which defaults to a single "\n".
+ */
+export declare function decodeQuotedPrintable(lines: AsyncIterable<Uint8Array>, newLine?: Uint8Array): AsyncIterableIterator<Uint8Array>;
+/**
+ * decoder for base64
+ */
+export declare function decodeBase64(lines: AsyncIterable<Uint8Array>): AsyncIterableIterator<Uint8Array>;
+/**
+ * decoder for 7bit and 8bit
+ */
+export declare function decodeIdentity(lines: AsyncIterable<Uint8Array>): AsyncIterableIterator<Uint8Array>;
+/**
+ * decoder for binary
+ *
+ * For implementation reasons, binary can't be supported, so we throw a special
+ * error.
+ */
+export declare function decodeBinary(): never;

package/dist/utils.js

@@ -0,0 +1,221 @@
+/**
+ * Utilities
+ *
+ * Most of these use Uint8Array explicitely because we don't care about a raw
+ * buffer, but about views of bytes, and this makes sure that we handle the
+ * types appropriately.
+ *
+ * @packageDocumentation
+ */
+import { decodeBase64 as db64 } from "./base64";
+/**
+ * create a Uint8Array from a buffer
+ *
+ * This method is necessary since creating a Uint8Array from a view will copy
+ * the elements instead of creating a view of the underling buffer.
+ */
+export function toBytes(buff) {
+    if (ArrayBuffer.isView(buff)) {
+        return new Uint8Array(buff.buffer, buff.byteOffset, buff.byteLength);
+    }
+    else {
+        return new Uint8Array(buff);
+    }
+}
+/**
+ * Compare two byte arrays for equality
+ */
+export function bytesEqual(left, right) {
+    if (left.length !== right.length) {
+        return false;
+    }
+    else if (left.byteOffset % 4 === right.byteOffset % 4) {
+        // words align, can use fast path
+        const begin = (4 - (left.byteOffset % 4)) % 4;
+        const fourLen = Math.floor((left.byteLength - begin) / 4);
+        const end = begin + fourLen * 4;
+        const left32 = new Uint32Array(left.buffer, left.byteOffset + begin, fourLen);
+        const right32 = new Uint32Array(right.buffer, right.byteOffset + begin, fourLen);
+        for (let i = 0; i < begin; ++i) {
+            if (left[i] !== right[i]) {
+                return false;
+            }
+        }
+        for (let i = 0; i < fourLen; ++i) {
+            if (left32[i] !== right32[i]) {
+                return false;
+            }
+        }
+        for (let i = end; i < left.length; ++i) {
+            if (left[i] !== right[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+    else {
+        // slower byte path
+        for (const ind of left.keys()) {
+            if (left[ind] !== right[ind]) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
+/**
+ * convert readable stream to async iterator
+ */
+export function asIterable(stream) {
+    return {
+        [Symbol.asyncIterator]() {
+            const reader = stream.getReader();
+            return {
+                // NOTE this complexity is for typescript
+                async next() {
+                    const res = await reader.read();
+                    if (res.done) {
+                        return { done: true, value: undefined };
+                    }
+                    else {
+                        return { done: false, value: res.value };
+                    }
+                },
+            };
+        },
+    };
+}
+/**
+ * Find index of one byte array in another
+ */
+export function indexOf(haystack, needle) {
+    return haystack.findIndex((val, ind) => {
+        if (val !== needle[0] || ind + needle.length > haystack.length) {
+            return false;
+        }
+        else {
+            for (let i = 1; i < needle.length; ++i) {
+                if (haystack[ind + i] !== needle[i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+    });
+}
+/**
+ * Split a stream of bytes
+ *
+ * Takes a stream of data modeled as an async iterator of ArrayBuffer for
+ * compatibility between node and web, and splits it into an async iterator
+ * where each value is delimited by the split sequence.
+ */
+export async function* splitStream(iter, split) {
+    let current = new Uint8Array(0);
+    for await (const chunk of iter) {
+        const next = toBytes(chunk);
+        current = current.length ? concat([current, next]) : next;
+        let nextInd;
+        while ((nextInd = indexOf(current, split)) !== -1) {
+            yield current.subarray(0, nextInd);
+            current = current.subarray(nextInd + split.length);
+        }
+    }
+    yield current;
+}
+/**
+ * concatenate multiple buffers
+ */
+function concat(chunks) {
+    const totalBytes = chunks.reduce((t, c) => t + c.length, 0);
+    const res = new Uint8Array(totalBytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+        res.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return res;
+}
+/**
+ * collect an async iterable of buffers into one
+ */
+export async function collect(stream) {
+    const chunks = [];
+    for await (const chunk of stream) {
+        chunks.push(chunk);
+    }
+    return concat(chunks);
+}
+// default new line character for quoted printable decoding
+const defaultNewLine = new Uint8Array([10]);
+/**
+ * decoder for quoted printable
+ *
+ * If quoted printable "lines" aren't escaped with an "=" then a new line needs
+ * to be inserted. We use `newLine` which defaults to a single "\n".
+ */
+export async function* decodeQuotedPrintable(lines, newLine = defaultNewLine) {
+    for await (const bytes of lines) {
+        const res = new Uint8Array(bytes.length + newLine.length);
+        let softLine = false; // if newline wasn't escaped so we need to add
+        let destInd = 0;
+        for (let ind = 0; ind < bytes.length; ++ind) {
+            const code = bytes[ind];
+            if (code >= 128) {
+                throw new Error(`got non-ascii character when decoding quoted printable: ${bytes}`);
+            }
+            if (code !== 61) {
+                res[destInd++] = code;
+            }
+            else {
+                // escaped char
+                const first = bytes[++ind];
+                if (first === undefined) {
+                    // soft newline
+                    softLine = true;
+                }
+                else {
+                    const second = bytes[++ind];
+                    if (second === undefined) {
+                        throw new Error("quoted printable escape (=) was not followed by two bytes");
+                    }
+                    let val = parseInt(String.fromCharCode(first), 16);
+                    val *= 16;
+                    val += parseInt(String.fromCharCode(second), 16);
+                    res[destInd++] = val;
+                }
+            }
+        }
+        if (!softLine) {
+            res.set(newLine, destInd);
+            destInd += newLine.length;
+        }
+        yield res.subarray(0, destInd);
+    }
+}
+const decoder = new TextDecoder();
+/**
+ * decoder for base64
+ */
+export async function* decodeBase64(lines) {
+    for await (const bytes of lines) {
+        yield db64(decoder.decode(bytes));
+    }
+}
+/**
+ * decoder for 7bit and 8bit
+ */
+export async function* decodeIdentity(lines) {
+    for await (const bytes of lines) {
+        yield bytes;
+    }
+}
+/**
+ * decoder for binary
+ *
+ * For implementation reasons, binary can't be supported, so we throw a special
+ * error.
+ */
+export function decodeBinary() {
+    throw new Error("binary transfer-encoding is explicitly not supported and trying to add an implementation will likely result in unexpected results, but if you want to ignore anyway, set binary to `decode8bit`");
+}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "mhtml-stream",
+  "version": "0.0.1",
+  "description": "Zero dependency stream MHTML parser",
+  "keywords": [
+    "mhtml",
+    "stream"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/erikbrinkman/mhtml-stream.git"
+  },
+  "author": "Erik Brinkman <erik.brinkman@gmail.com>",
+  "license": "MIT",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs.min.js",
+  "unpkg": "dist/index.iife.min.js",
+  "files": [
+    "/dist/**/*.js",
+    "/dist/**/*.d.ts"
+  ],
+  "type": "module",
+  "scripts": {
+    "fmt": "prettier --write 'src/*.ts' '*.json'",
+    "lint": "tsc && eslint 'src/*.ts'",
+    "test": "jest --coverage",
+    "doc": "typedoc",
+    "build": "tsc -p tsconfig.build.json && esbuild src/index.ts --bundle --minify --format=cjs --outfile=dist/index.cjs.min.js && esbuild src/index.ts --bundle --minify --global-name=mhtmlStream --outfile=dist/index.iife.min.js",
+    "prepare": "yarn lint && yarn test && yarn build"
+  },
+  "devDependencies": {
+    "@types/node": "^17.0.14",
+    "esbuild": "^0.14.17",
+    "@babel/core": "^7.16.12",
+    "@babel/preset-env": "^7.16.11",
+    "@babel/preset-typescript": "^7.16.7",
+    "@types/jest": "^27.4.0",
+    "@typescript-eslint/eslint-plugin": "^5.10.0",
+    "@typescript-eslint/parser": "^5.10.0",
+    "babel-jest": "^27.4.6",
+    "eslint": "^8.7.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-jest": "^26.0.0",
+    "eslint-plugin-spellcheck": "^0.0.19",
+    "jest": "^27.4.7",
+    "prettier": "^2.5.1",
+    "prettier-plugin-organize-imports": "^2.3.4",
+    "typedoc": "^0.22.11",
+    "typescript": "^4.5.5"
+  }
+}