npm - @bitblit/ratchet-node-only - Versions diffs - 4.0.475-alpha → 5.0.107 - Mend

@bitblit/ratchet-node-only 4.0.475-alpha → 5.0.107

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (22) hide show

package/lib/third-party/common-crawl/common-crawl-service.d.ts ADDED Viewed

@@ -0,0 +1,15 @@
+import { WarcEntry } from './model/warc-entry.js';
+import { CommonCrawlScan } from './model/common-crawl-scan.js';
+import { CommonCrawlFetchOptions } from './model/common-crawl-fetch-options.js';
+import { DomainIndexEntryRaw } from './model/domain-index-entry-raw.js';
+import { IndexEntryRaw } from './model/index-entry-raw.js';
+export declare class CommonCrawlService {
+    static readonly COMMON_CRAWL_URL: string;
+    static readonly CURRENT_CRAWL: string;
+    fetchIndexes(): Promise<IndexEntryRaw[]>;
+    readPageData(entry: DomainIndexEntryRaw): Promise<any>;
+    static validLanguages(entry: DomainIndexEntryRaw): string[];
+    pullPageEntry(entry: DomainIndexEntryRaw, language?: string): Promise<WarcEntry>;
+    search(options: CommonCrawlFetchOptions): Promise<DomainIndexEntryRaw[]>;
+    scanSite(opts: CommonCrawlFetchOptions, onPage?: (idx: number, cnt: number, header: string) => Promise<any>): Promise<CommonCrawlScan>;
+}

package/lib/third-party/common-crawl/common-crawl-service.js ADDED Viewed

@@ -0,0 +1,153 @@
+import fetch from 'cross-fetch';
+import * as querystring from 'node:querystring';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import zlib from 'zlib';
+import { Readable } from 'stream';
+import warc from 'warc';
+import * as cheerio from 'cheerio';
+import { ErrorRatchet } from "@bitblit/ratchet-common/lang/error-ratchet";
+import { NodeStreamRatchet } from "../../stream/node-stream-ratchet";
+import { Logger } from "@bitblit/ratchet-common/logger/logger";
+import { PromiseRatchet } from "@bitblit/ratchet-common/lang/promise-ratchet";
+import { StopWatch } from "@bitblit/ratchet-common/lang/stop-watch";
+export class CommonCrawlService {
+    static COMMON_CRAWL_URL = 'https://index.commoncrawl.org/';
+    static CURRENT_CRAWL = 'CC-MAIN-2024-33';
+    async fetchIndexes() {
+        const res = await fetch(CommonCrawlService.COMMON_CRAWL_URL + 'collinfo.json');
+        const output = await res.json();
+        return output;
+    }
+    async readPageData(entry) {
+        const rval = {};
+        const langs = CommonCrawlService.validLanguages(entry);
+        for (const lang of langs) {
+            rval[lang] = [];
+            const data = await this.pullPageEntry(entry);
+            const asString = data.content.toString();
+            const parsed = cheerio.load(asString);
+            ['p', 'div', 'span'].forEach((tag) => {
+                parsed(tag).each((idx, el) => {
+                    const txt = StringRatchet.trimToNull(parsed(el).text());
+                    if (txt && txt.includes('.')) {
+                        rval[lang].push(txt);
+                    }
+                });
+            });
+        }
+        return rval;
+    }
+    static validLanguages(entry) {
+        const validLangs = entry.languages
+            .split(',')
+            .map((s) => StringRatchet.trimToNull(s))
+            .filter((s) => !!s);
+        return validLangs;
+    }
+    async pullPageEntry(entry, language) {
+        const prefix = 'https://data.commoncrawl.org/';
+        const url = prefix + entry.filename;
+        const headers = { Range: 'bytes=' + entry.offset + '-' + (entry.offset + entry.length + 1) };
+        if (language) {
+            if (!CommonCrawlService.validLanguages(entry).includes(language)) {
+                throw ErrorRatchet.fErr('Requested language %s, but valid are %s', language, entry.languages);
+            }
+            headers['Accept-Language'] = language;
+        }
+        const resp = await fetch(url, {
+            headers: headers,
+        });
+        let reader = null;
+        if (resp.body instanceof Readable) {
+            reader = resp.body;
+        }
+        else if (resp.body instanceof ReadableStream) {
+            reader = NodeStreamRatchet.webReadableStreamToNodeReadable(resp.body);
+        }
+        const warcstream = new warc();
+        Logger.info('Headers is %j', resp.headers);
+        let rval = null;
+        reader
+            .pipe(zlib.createGunzip())
+            .pipe(warcstream)
+            .on('data', (val) => {
+            Logger.info('Got data ' + val.content.length);
+            rval = val;
+            warcstream.destroy();
+        })
+            .on('close', () => {
+            Logger.info('Got close event');
+        })
+            .on('error', (err) => {
+            Logger.error('Read error: %s', err, err);
+        });
+        while (!rval) {
+            await PromiseRatchet.wait(500);
+        }
+        const conv = {
+            protocol: rval.protocol,
+            headers: rval.headers,
+            content: rval.content.toString(),
+        };
+        return conv;
+    }
+    async search(options) {
+        RequireRatchet.notNullOrUndefined(options, 'options');
+        RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(options.url, 'options.url');
+        let url = CommonCrawlService.COMMON_CRAWL_URL + (options.index || CommonCrawlService.CURRENT_CRAWL);
+        url += '-index?';
+        const params = {
+            url: options.url,
+            matchType: options.matchType || 'domain',
+            output: 'json',
+        };
+        const urlPart = querystring.stringify(params);
+        url += urlPart;
+        const res = await fetch(url);
+        const body = await res.text();
+        let rval = null;
+        if (res.status === 200) {
+            const lines = body.split('\n');
+            rval = lines.map((s) => (StringRatchet.trimToNull(s) ? JSON.parse(s) : null)).filter((s) => !!s);
+        }
+        else {
+            Logger.error('Failed to fetch: %s : %s : %j : %s', res.status, res.statusText, res.headers, res.body);
+        }
+        return rval;
+    }
+    async scanSite(opts, onPage) {
+        const sw = new StopWatch();
+        const rval = {
+            options: opts,
+            pageIndexes: [],
+            parsed: [],
+            errors: [],
+        };
+        Logger.info('Performing domain index scan with %j', opts);
+        rval.pageIndexes = await this.search(opts);
+        Logger.info('Found %d entries, pulling each', rval.pageIndexes.length);
+        for (const [idx, ent] of rval.pageIndexes.entries()) {
+            try {
+                Logger.info('Pulling item %d of %d, %s', idx, rval.pageIndexes.length, sw.dumpExpected(idx / rval.pageIndexes.length));
+                if (onPage) {
+                    try {
+                        await onPage(idx, rval.pageIndexes.length, ent.url);
+                    }
+                    catch (err) {
+                        Logger.warn('Failed onpage: %s', err);
+                    }
+                }
+                const parsed = await this.pullPageEntry(ent);
+                rval.parsed.push(parsed);
+            }
+            catch (err) {
+                Logger.warn('Failed to pull %j : %s', ent, err);
+                rval.errors.push({ pageIdx: ent, error: err });
+            }
+        }
+        Logger.info('Completed full scan in %s', sw.dump());
+        return rval;
+    }
+}
+//# sourceMappingURL=common-crawl-service.js.map

package/lib/third-party/common-crawl/common-crawl-service.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"common-crawl-service.js","sourceRoot":"","sources":["../../../src/third-party/common-crawl/common-crawl-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC;AAC5E,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAOnC,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,uCAAuC,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAKpE,MAAM,OAAO,kBAAkB;IACtB,MAAM,CAAU,gBAAgB,GAAW,gCAAgC,CAAC;IAC5E,MAAM,CAAU,aAAa,GAAW,iBAAiB,CAAC;IAE1D,KAAK,CAAC,YAAY;QACvB,MAAM,GAAG,GAAa,MAAM,KAAK,CAAC,kBAAkB,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC;QACzF,MAAM,MAAM,GAAoB,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,KAA0B;QAClD,MAAM,IAAI,GAA6B,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAa,kBAAkB,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEjE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,QAAQ,GAAW,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjD,MAAM,MAAM,GAAiB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAW,EAAE,EAAmB,EAAE,EAAE;oBACpD,MAAM,GAAG,GAAW,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;oBAChE,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC7B,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAEvB,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,KAA0B;QACrD,MAAM,UAAU,GAAa,KAAK,CAAC,SAAS;aACzC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;aACvC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,UAAU,CAAC;IACpB,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,KAA0B,EAAE,QAAiB;QACtE,MAAM,MAAM,GAAW,+BAA+B,CAAC;QACvD,MAAM,GAAG,GAAW,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC;QAE5C,MAAM,OAAO,GAA2B,EAAE,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACrH,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjE,MAAM,YAAY,CAAC,IAAI,CAAC,yCAAyC,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAChG,CAAC;YACD,OAAO,CAAC,iBAAiB,CAAC,GAAG,QAAQ,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,GAAa,MAAM,KAAK,CAAC,GAAG,EAAE;YACtC,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAGH,IAAI,MAAM,GAAa,IAAI,CAAC;QAC5B,IAAI,IAAI,CAAC,IAAI,YAAY,QAAQ,EAAE,CAAC;YAClC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;QACrB,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,YAAY,cAAc,EAAE,CAAC;YAC/C,MAAM,GAAG,iBAAiB,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,UAAU,GAAS,IAAI,IAAI,EAAE,CAAC;QAOpC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAE3C,IAAI,IAAI,GAAiB,IAAI,CAAC;QAE9B,MAAM;aACH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;aACzB,IAAI,CAAC,UAAU,CAAC;aAChB,EAAE,CAAC,MAAM,EAAE,CAAC,GAAiB,EAAE,EAAE;YAChC,MAAM,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,GAAG,GAAG,CAAC;YAIX,UAAU,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC,CAAC;aACD,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAChB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAEjC,CAAC,CAAC;aACD,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACnB,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAE3C,CAAC,CAAC,CAAC;QAEL,OAAO,CAAC,IAAI,EAAE,CAAC;YAEb,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAc;YACtB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;SACjC,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,OAAgC;QAClD,cAAc,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACtD,cAAc,CAAC,sCAAsC,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAElF,IAAI,GAAG,GAAW,kBAAkB,CAAC,gBAAgB,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC5G,GAAG,IAAI,SAAS,CAAC;QACjB,MAAM,MAAM,GAAG;YACb,GAAG,EAAE,OAAO,CAAC,GAAG;YAGhB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ;YAMxC,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,OAAO,GAAW,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtD,GAAG,IAAI,OAAO,CAAC;QAIf,MAAM,GAAG,GAAa,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,IAAI,GAAW,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,GAA0B,IAAI,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnG,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACxG,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAeM,KAAK,CAAC,QAAQ,CACnB,IAA6B,EAC7B,MAAmE;QAEnE,MAAM,EAAE,GAAc,IAAI,SAAS,EAAE,CAAC;QACtC,MAAM,IAAI,GAAoB;YAC5B,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACvE,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;gBACvH,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC;wBACH,MAAM,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;oBACtD,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;gBACD,MAAM,MAAM,GAAc,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAExD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC"}

package/lib/third-party/common-crawl/model/common-crawl-fetch-options.d.ts ADDED Viewed

@@ -0,0 +1,12 @@
+export interface CommonCrawlFetchOptions {
+    url: string;
+    index?: string;
+    from?: string;
+    to?: string;
+    showNumPages?: boolean;
+    matchType?: 'exact' | 'prefix' | 'host' | 'domain';
+    limit?: number;
+    sort?: 'asc' | 'desc';
+    page?: number;
+    pageSize?: number;
+}

package/lib/third-party/common-crawl/model/common-crawl-fetch-options.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=common-crawl-fetch-options.js.map

package/lib/third-party/common-crawl/model/common-crawl-fetch-options.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"common-crawl-fetch-options.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/common-crawl-fetch-options.ts"],"names":[],"mappings":""}

package/lib/third-party/common-crawl/model/common-crawl-scan.d.ts ADDED Viewed

@@ -0,0 +1,9 @@
+import { WarcEntry } from './warc-entry.js';
+import { DomainIndexEntryRaw } from './domain-index-entry-raw.js';
+import { CommonCrawlFetchOptions } from './common-crawl-fetch-options.js';
+export interface CommonCrawlScan {
+    options: CommonCrawlFetchOptions;
+    pageIndexes: DomainIndexEntryRaw[];
+    parsed: WarcEntry[];
+    errors: any[];
+}

package/lib/third-party/common-crawl/model/common-crawl-scan.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=common-crawl-scan.js.map

package/lib/third-party/common-crawl/model/common-crawl-scan.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"common-crawl-scan.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/common-crawl-scan.ts"],"names":[],"mappings":""}

package/lib/third-party/common-crawl/model/domain-index-entry-raw.d.ts ADDED Viewed

@@ -0,0 +1,14 @@
+export interface DomainIndexEntryRaw {
+    urlkey: string;
+    timestamp: string;
+    url: string;
+    mime: string;
+    'mime-detected': string;
+    status: string;
+    digest: string;
+    length: string;
+    offset: string;
+    filename: string;
+    languages: string;
+    encoding: string;
+}

package/lib/third-party/common-crawl/model/domain-index-entry-raw.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=domain-index-entry-raw.js.map

package/lib/third-party/common-crawl/model/domain-index-entry-raw.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"domain-index-entry-raw.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/domain-index-entry-raw.ts"],"names":[],"mappings":""}

package/lib/third-party/common-crawl/model/index-entry-raw.d.ts ADDED Viewed

@@ -0,0 +1,8 @@
+export interface IndexEntryRaw {
+    id: string;
+    name: string;
+    timegate: string;
+    'cdx-api': string;
+    from: string;
+    to: string;
+}

package/lib/third-party/common-crawl/model/index-entry-raw.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=index-entry-raw.js.map

package/lib/third-party/common-crawl/model/index-entry-raw.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"index-entry-raw.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/index-entry-raw.ts"],"names":[],"mappings":""}

package/lib/third-party/common-crawl/model/warc-entry-raw.d.ts ADDED Viewed

@@ -0,0 +1,5 @@
+export interface WarcEntryRaw {
+    protocol: string;
+    headers: Record<string, string>;
+    content: Buffer;
+}

package/lib/third-party/common-crawl/model/warc-entry-raw.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=warc-entry-raw.js.map

package/lib/third-party/common-crawl/model/warc-entry-raw.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"warc-entry-raw.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/warc-entry-raw.ts"],"names":[],"mappings":""}

package/lib/third-party/common-crawl/model/warc-entry.d.ts ADDED Viewed

@@ -0,0 +1,5 @@
+export interface WarcEntry {
+    protocol: string;
+    headers: Record<string, string>;
+    content: string;
+}

package/lib/third-party/common-crawl/model/warc-entry.js ADDED Viewed

	@@ -0,0 +1,2 @@
1	+ export {};
2	+ //# sourceMappingURL=warc-entry.js.map

package/lib/third-party/common-crawl/model/warc-entry.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"warc-entry.js","sourceRoot":"","sources":["../../../../src/third-party/common-crawl/model/warc-entry.ts"],"names":[],"mappings":""}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-node-only",
-  "version": "4.0.475-alpha",
+  "version": "5.0.107",
   "description": "Ratchet tools for use on node-only",
   "note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
   "sideEffects": false,
@@ -55,20 +55,27 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@bitblit/ratchet-common": "4.0.475-alpha"
+    "@bitblit/ratchet-common": "5.0.107"
   },
   "optionalDependencies": {
+    "cheerio": "1.0.0",
     "csv": "6.3.10",
     "jsonwebtoken": "9.0.2",
-    "rxjs": "7.8.1"
+    "rxjs": "7.8.1",
+    "warc": "1.0.1"
   },
   "peerDependencies": {
-    "@bitblit/ratchet-common": "4.0.475-alpha",
+    "@bitblit/ratchet-common": "5.0.107",
+    "cheerio": "^1.0.0",
     "csv": "^6.3.10",
     "jsonwebtoken": "^9.0.2",
-    "rxjs": "^7.8.1"
+    "rxjs": "^7.8.1",
+    "warc": "^1.0.1"
   },
   "peerDependenciesMeta": {
+    "cheerio": {
+      "optional": true
+    },
     "csv": {
       "optional": true
     },
@@ -77,6 +84,12 @@
     },
     "rxjs": {
       "optional": true
+    },
+    "warc": {
+      "optional": true
     }
+  },
+  "devDependencies": {
+    "@types/cheerio": "0.22.35"
   }
 }