@bitblit/ratchet-node-only 6.0.145-alpha → 6.0.147-alpha

package/src/jwt/jwt-ratchet.spec.ts

@@ -0,0 +1,85 @@
+import { JwtRatchet } from './jwt-ratchet.js';
+import { describe, expect, test } from 'vitest';
+import { JwtRatchetConfig } from './jwt-ratchet-config.js';
+import { ExpiredJwtHandling } from '@bitblit/ratchet-common/jwt/expired-jwt-handling';
+
+describe('#jwtRatchet', function () {
+  function createConfig(): JwtRatchetConfig {
+    const jwtRatchetConfig: JwtRatchetConfig = {
+      encryptionKeyPromise: Promise.resolve('test1234'),
+      decryptKeysPromise: Promise.resolve([]),
+      jtiGenerator: undefined,
+      decryptOnlyKeyUseLogLevel: undefined,
+      parseFailureLogLevel: undefined,
+    };
+    return jwtRatchetConfig;
+  }
+
+  test('should test expiration flag for a token with millisecond expiration', async () => {
+    const jwt: JwtRatchet = new JwtRatchet(createConfig());
+
+    const token1: string = await jwt.createTokenString({ test: 1, exp: Date.now() - 100 }, null);
+
+    const output: any = await jwt.decodeToken(token1, ExpiredJwtHandling.ADD_FLAG);
+    expect(output).not.toBeNull();
+    expect(JwtRatchet.hasExpiredFlag(output)).toBeTruthy();
+  });
+
+  test('should test expiration calculation for a token', async () => {
+    const jwt: JwtRatchet = new JwtRatchet(createConfig());
+
+    const token1: string = await jwt.createTokenString({ test: 1 }, 120);
+    const output: number = await JwtRatchet.secondsRemainingUntilExpiration(token1);
+
+    expect(output).not.toBeNull();
+    expect(output).toBeLessThan(121);
+    expect(output).toBeGreaterThan(115);
+  });
+
+  test('should test round-trip for a token', async () => {
+    const jwt: JwtRatchet = new JwtRatchet(createConfig());
+
+    const token1: string = await jwt.createTokenString({ test: 1 }, 120);
+    const output: any = await jwt.decodeToken(token1);
+
+    expect(output).not.toBeNull();
+    expect(output['test']).toEqual(1);
+  });
+
+  test('should test round-trip for a token with array enc keys', async () => {
+    const subCfg: JwtRatchetConfig = createConfig();
+    subCfg.encryptionKeyPromise = Promise.resolve(['test1234', 'test5678']);
+    const jwt: JwtRatchet = new JwtRatchet(subCfg);
+
+    const token1: string = await jwt.createTokenString({ test: 1 }, 120);
+    const token2: string = await jwt.createTokenString({ test: 1 }, 120);
+    const token3: string = await jwt.createTokenString({ test: 1 }, 120);
+    const output1: any = await jwt.decodeToken(token1);
+    const output2: any = await jwt.decodeToken(token2);
+    const output3: any = await jwt.decodeToken(token3);
+
+    expect(output1).not.toBeNull();
+    expect(output1['test']).toEqual(1);
+    expect(output2).not.toBeNull();
+    expect(output2['test']).toEqual(1);
+    expect(output3).not.toBeNull();
+    expect(output3['test']).toEqual(1);
+  });
+
+  test('should decode with a decode key', async () => {
+    const oldCfg: JwtRatchetConfig = createConfig();
+    oldCfg.encryptionKeyPromise = Promise.resolve('oldKey');
+    const newCfg: JwtRatchetConfig = createConfig();
+    newCfg.encryptionKeyPromise = Promise.resolve('newKey');
+    newCfg.decryptKeysPromise = Promise.resolve(['oldKey']);
+
+    const jwtOld: JwtRatchet = new JwtRatchet(oldCfg);
+    const jwtNew: JwtRatchet = new JwtRatchet(newCfg);
+
+    const token1: string = await jwtOld.createTokenString({ test: 1 }, 120);
+    const output: any = await jwtNew.decodeToken(token1);
+
+    expect(output).not.toBeNull();
+    expect(output['test']).toEqual(1);
+  });
+});

package/src/jwt/jwt-ratchet.ts

@@ -0,0 +1,204 @@
+import { JwtRatchetLike } from './jwt-ratchet-like.js';
+import { JwtRatchetConfig } from './jwt-ratchet-config.js';
+import jsonwebtoken from 'jsonwebtoken';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { LoggerLevelName } from '@bitblit/ratchet-common/logger/logger-level-name';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { JwtPayloadExpirationRatchet } from '@bitblit/ratchet-common/jwt/jwt-payload-expiration-ratchet';
+import { JwtTokenBase } from '@bitblit/ratchet-common/jwt/jwt-token-base';
+import { ExpiredJwtHandling } from '@bitblit/ratchet-common/jwt/expired-jwt-handling';
+
+/**
+ * Functions to help with creating and decoding JWTs
+ *
+ * JWTRatchet accepts promises for its inputs for the simple reason that best practice dictates that the keys
+ * should never be in the code, which means it is likely somewhere else.  That MIGHT be somewhere synchronous
+ * like an environmental variable, but it could very likely be someplace remote like a secure key store.  By
+ * accepting promises here, we make it easy to do JwtRatchet construction in a place (like an IOT container)
+ * that itself must be synchronous
+ */
+export class JwtRatchet implements JwtRatchetLike {
+  constructor(private cfg: JwtRatchetConfig) {
+    RequireRatchet.notNullOrUndefined(cfg, 'config');
+    RequireRatchet.notNullOrUndefined(cfg.encryptionKeyPromise, 'encryptionKeyPromise');
+
+    cfg.jtiGenerator = cfg.jtiGenerator ?? StringRatchet.createType4Guid;
+    cfg.decryptOnlyKeyUseLogLevel = cfg.decryptOnlyKeyUseLogLevel ?? LoggerLevelName.info;
+    cfg.parseFailureLogLevel = cfg.parseFailureLogLevel ?? LoggerLevelName.debug;
+  }
+
+  public get copyConfig(): JwtRatchetConfig {
+    const rval: JwtRatchetConfig = {
+      encryptionKeyPromise: this.cfg.encryptionKeyPromise,
+      decryptKeysPromise: this.cfg.decryptKeysPromise,
+      jtiGenerator: this.cfg.jtiGenerator,
+      decryptOnlyKeyUseLogLevel: this.cfg.decryptOnlyKeyUseLogLevel,
+      parseFailureLogLevel: this.cfg.parseFailureLogLevel,
+    };
+    return rval;
+  }
+
+  public get encryptionKeyPromise(): Promise<string | string[]> {
+    return this.cfg.encryptionKeyPromise;
+  }
+
+  public get decryptKeysPromise(): Promise<string[]> {
+    return this.cfg.decryptKeysPromise;
+  }
+
+  public get jtiGenerator(): () => string {
+    return this.cfg.jtiGenerator;
+  }
+
+  public get decryptOnlyKeyUseLogLevel(): LoggerLevelName {
+    return this.cfg.decryptOnlyKeyUseLogLevel;
+  }
+
+  public get parseFailureLogLevel(): LoggerLevelName {
+    return this.cfg.parseFailureLogLevel;
+  }
+
+  public static async invalidSafeDecode<T>(
+    payloadString: string,
+    decryptKey: string,
+    logLevel: LoggerLevelName = LoggerLevelName.silly,
+  ): Promise<T> {
+    let rval: T = null;
+    try {
+      rval = jsonwebtoken.verify(payloadString, decryptKey, { ignoreExpiration: true }) as unknown as T; // We'll check/flag expiration later
+    } catch (err) {
+      Logger.logByLevel(logLevel, 'Caught %s - ignoring', err);
+    }
+    return rval;
+  }
+
+  public static async secondsRemainingUntilExpiration(payloadString: string): Promise<number> {
+    let rval: number = null;
+    if (StringRatchet.trimToNull(payloadString)) {
+      const output: JwtTokenBase = await JwtRatchet.decodeTokenNoVerify<any>(payloadString);
+      const nowSecond: number = Math.floor(Date.now() / 1000);
+      if (output.exp) {
+        // A backwards compatibility hack since some of my old code used to incorrectly write the exp field in milliseconds
+        const expSeconds: number = output.exp > nowSecond * 100 ? Math.floor(output.exp / 1000) : output.exp;
+        rval = Math.max(0, expSeconds - nowSecond);
+      }
+    }
+    return rval;
+  }
+
+  public static async msRemainingUntilExpiration(payloadString: string): Promise<number> {
+    const secs: number = await JwtRatchet.secondsRemainingUntilExpiration(payloadString);
+    let rval: number = null;
+    if (secs !== null && secs !== undefined) {
+      rval = secs * 1000;
+    }
+    return rval;
+  }
+
+  public async decodeToken<T extends JwtTokenBase>(
+    payloadString: string,
+    expiredHandling: ExpiredJwtHandling = ExpiredJwtHandling.RETURN_NULL,
+  ): Promise<T> {
+    const encKeys: string[] = await this.encryptionKeyArray();
+    let decKeys: string[] = Object.assign([], encKeys);
+    if (this.decryptKeysPromise) {
+      decKeys = decKeys.concat(await this.decryptKeysPromise);
+    }
+
+    const keysTried: string[] = []; //[StringRatchet.obscure(encKey, 1, 1)];
+    let payload: T = null; //await JwtRatchet.invalidSafeDecode(payloadString, encKey);
+
+    for (let i = 0; i < decKeys.length && !payload; i++) {
+      keysTried.push(StringRatchet.obscure(decKeys[i], 1, 1));
+      // Only Log on the last one since it might have just been an old key
+      const logLevel: LoggerLevelName =
+        i === decKeys.length - 1 && this.parseFailureLogLevel ? this.parseFailureLogLevel : LoggerLevelName.silly;
+      payload = await JwtRatchet.invalidSafeDecode(payloadString, decKeys[i], logLevel);
+      if (payload && i >= encKeys.length) {
+        Logger.logByLevel(this.decryptOnlyKeyUseLogLevel, 'Used old key to decode token : %s', StringRatchet.obscure(decKeys[i], 2));
+      }
+    }
+
+    if (payload) {
+      payload = JwtPayloadExpirationRatchet.processPayloadExpiration(payload, expiredHandling);
+    } else {
+      Logger.warn('Unable to parse a payload (Tried %j) from : %s', keysTried, payloadString);
+    }
+
+    return payload;
+  }
+
+  public async encryptionKeyArray(): Promise<string[]> {
+    const encKey: string | string[] = await this.encryptionKeyPromise;
+    const rval: string[] = Array.isArray(encKey) ? encKey : [encKey];
+    if (rval.length < 1) {
+      throw new Error('Cannot create JwtRatchet with empty encryption key set');
+    }
+    return rval;
+  }
+
+  public async selectRandomEncryptionKey(): Promise<string> {
+    const encKey: string[] = await this.encryptionKeyArray();
+    const rval: string = encKey[Math.floor(Math.random() * encKey.length)];
+    return rval;
+  }
+
+  public async createTokenString(payload: any, expirationSeconds?: number, overrideEncryptionKey?: string): Promise<string> {
+    const encKey: string = StringRatchet.trimToNull(overrideEncryptionKey)
+      ? StringRatchet.trimToNull(overrideEncryptionKey)
+      : await this.selectRandomEncryptionKey();
+
+    RequireRatchet.notNullOrUndefined(payload, 'payload');
+    payload.jti = this.jtiGenerator ? this.jtiGenerator() : null; // Setup unique code
+    if (expirationSeconds) {
+      const nowSeconds = Math.floor(Date.now() / 1000);
+      const expires = nowSeconds + expirationSeconds;
+      Logger.debug('Forcing expiration to %d', expires);
+      payload.exp = expires;
+    }
+
+    const token: string = jsonwebtoken.sign(payload, encKey); // , algorithm = 'HS256')
+    return token;
+  }
+
+  public async refreshJWTString(tokenString: string, allowExpired?: boolean, expirationSeconds?: number): Promise<string> {
+    const handling: ExpiredJwtHandling = allowExpired ? ExpiredJwtHandling.ADD_FLAG : ExpiredJwtHandling.THROW_EXCEPTION;
+    const payload: JwtTokenBase = await this.decodeToken(tokenString, handling);
+
+    const originalDurationSeconds: number = payload.exp && payload.iat ? payload.exp - payload.iat : null;
+    const newExpirationSeconds: number = expirationSeconds || originalDurationSeconds;
+    // Remove any old stuff
+    JwtRatchet.removeJwtFields(payload);
+    JwtRatchet.removeExpiredFlag(payload); // If it wasnt allowed an exception was thrown above anyway
+    const token: string = await this.createTokenString(payload, newExpirationSeconds);
+    return token;
+  }
+
+  // Helper method that reads the token without checking it, therefore the keys are not needed
+  public static async decodeTokenNoVerify<T extends JwtTokenBase>(token: string): Promise<T> {
+    const rval: T = jsonwebtoken.decode(token) as T;
+    return rval;
+  }
+
+  // Removes any jwt fields from an object
+  public static removeJwtFields(ob: any) {
+    if (ob) {
+      ['iss', 'sub', 'aud', 'exp', 'nbf', 'iat', 'jti'].forEach((k) => {
+        // This isn't really dynamic
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete ob[k];
+      });
+    }
+  }
+
+  public static hasExpiredFlag(ob: any): boolean {
+    // Delegate for backwards compatibility
+    return JwtPayloadExpirationRatchet.hasExpiredFlag(ob);
+  }
+
+  public static removeExpiredFlag(ob: any) {
+    // Delegate for backwards compatibility
+    return JwtPayloadExpirationRatchet.removeExpiredFlag(ob);
+  }
+}

package/src/stream/buffer-writable.ts

@@ -0,0 +1,16 @@
+import { Writable } from 'stream';
+
+export class BufferWritable extends Writable {
+  private _val: any[] = [];
+
+  _write(chunk: any, encoding: string, callback): void {
+    if (chunk) {
+      this._val.push(chunk);
+    }
+    callback();
+  }
+
+  public get value(): Buffer {
+    return Buffer.concat(this._val);
+  }
+}

package/src/stream/multi-stream.ts

@@ -0,0 +1,15 @@
+import { Readable, ReadableOptions } from 'stream';
+
+export class MultiStream extends Readable {
+  _object: any;
+
+  constructor(object: any, options: ReadableOptions = {}) {
+    super(object instanceof Buffer || typeof object === 'string' ? options : { objectMode: true });
+    this._object = object;
+  }
+
+  _read() {
+    this.push(this._object);
+    this._object = null;
+  }
+}

package/src/stream/node-stream-ratchet.spec.ts

@@ -0,0 +1,19 @@
+import { Readable } from 'stream';
+import { NodeStreamRatchet } from './node-stream-ratchet.js';
+import { describe, expect, test } from 'vitest';
+
+describe('#NodeStreamRatchet', function () {
+  test('should wrap a string in a readable', async () => {
+    const r: Readable = NodeStreamRatchet.stringToReadable('test');
+    const out: Buffer = r.read(200);
+
+    expect(out.length).toEqual(4);
+  });
+
+  test('should wrap an number in a readable', async () => {
+    const r: Readable = NodeStreamRatchet.anyToStringReadable(401);
+    const out: Buffer = r.read(200);
+
+    expect(out.length).toEqual(3);
+  });
+});

package/src/stream/node-stream-ratchet.ts

@@ -0,0 +1,70 @@
+import { Readable, Writable } from 'stream';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+
+/**
+ * This class is specifically dealing with node streams as opposed to web streams
+ * (ie, Readable vs ReadableStream, Writeable vs WritableStream)
+ * https://stackoverflow.com/questions/61232291/difference-between-web-streams-and-node-js-stream-apis
+ *
+ * This is the only class that supports conversion between the two since the web will be assumed
+ * to not have access to the NodeJS classes, but Node DOES have access to the web classes
+ */
+export class NodeStreamRatchet {
+  // Empty constructor prevents instantiation
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() {}
+
+  public static readableToBufferSync(stream: Readable): Buffer {
+    const bufs = [];
+    let next: any = stream.read();
+    while (next) {
+      bufs.push(next);
+      next = stream.read();
+    }
+    return Buffer.concat(bufs);
+  }
+
+  public static stringToReadable(input: string): Readable {
+    return new Readable({
+      read() {
+        this.push(input);
+        this.push(null);
+      },
+    });
+  }
+
+  public static anyToStringReadable(input: any): Readable {
+    return input === null || input === undefined
+      ? NodeStreamRatchet.stringToReadable(null)
+      : NodeStreamRatchet.stringToReadable(StringRatchet.safeString(input));
+  }
+
+  // window.ReadableStream to Node.js Readable
+  public static webReadableStreamToNodeReadable(rs: ReadableStream): Readable {
+    const reader = rs.getReader();
+    const out = new Readable();
+    reader.read().then(async ({ value, done }) => {
+      while (!done) {
+        out.push(value);
+        ({ done, value } = await reader.read());
+      }
+      out.push(null);
+    });
+    return out;
+  }
+
+  // window.WritableStream to Node.js Writable
+  public static webWritableStreamToNodeWriteable(ws: WritableStream): Writable {
+    const writer = ws.getWriter();
+    const out = new Writable();
+    out._write = (chunk, encoding, callback) => {
+      writer.write(chunk);
+      callback();
+    };
+    out._final = (callback) => {
+      writer.close();
+      callback();
+    };
+    return out;
+  }
+}

package/src/stream/string-writable.spec.ts

@@ -0,0 +1,16 @@
+import { StringWritable } from './string-writable.js';
+import { describe, expect, test } from 'vitest';
+
+describe('#StringWritableStream', function () {
+  test('should write cumulatively to a string', async () => {
+    const sr: StringWritable = new StringWritable();
+    const callback = () => {
+      // Ignore me
+    };
+    sr._write('a', null, callback);
+    sr._write('b', null, callback);
+    sr._write('c', null, callback);
+
+    expect(sr.value.length).toEqual(3);
+  });
+});

package/src/stream/string-writable.ts

@@ -0,0 +1,14 @@
+import { Writable } from 'stream';
+
+export class StringWritable extends Writable {
+  private _val: string = '';
+
+  _write(chunk: any, encoding: string, callback): void {
+    this._val += chunk ? chunk.toString() : '';
+    callback();
+  }
+
+  public get value(): string {
+    return this._val;
+  }
+}

package/src/third-party/angular/angular-aot-rollup-plugin.ts

@@ -0,0 +1,18 @@
+import { execSync } from 'child_process';
+
+// Custom Rollup plugin to run Angular AOT
+export default function angularAotPlugin() {
+  return {
+    name: 'angular-aot-rollup-plugin',
+    buildStart() {
+      console.log('Running Angular AOT Compilation...');
+      try {
+        execSync('ngc -p tsconfig.json', { stdio: 'inherit' });
+      } catch (error) {
+        this.error('AOT Compilation failed.', error);
+      }
+    },
+  };
+}
+
+//module.exports = angularAotPlugin;

package/src/third-party/common-crawl/common-crawl-service.ts

@@ -0,0 +1,220 @@
+import fetch from 'cross-fetch';
+import * as querystring from 'node:querystring';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import zlib from 'zlib';
+import { Readable } from 'stream';
+import warc from 'warc';
+import * as cheerio from 'cheerio';
+import { WarcEntry } from './model/warc-entry.js';
+import { CommonCrawlScan } from './model/common-crawl-scan.js';
+import { CommonCrawlFetchOptions } from './model/common-crawl-fetch-options.js';
+import { DomainIndexEntryRaw } from './model/domain-index-entry-raw.js';
+import { IndexEntryRaw } from './model/index-entry-raw.js';
+import { WarcEntryRaw } from './model/warc-entry-raw.js';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import { NodeStreamRatchet } from '../../stream/node-stream-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
+import { CheerioAPI } from "cheerio";
+
+/**
+ * A very early take to simplify accessing and using the common crawl
+ */
+export class CommonCrawlService {
+  public static readonly COMMON_CRAWL_URL: string = 'https://index.commoncrawl.org/';
+  public static readonly CURRENT_CRAWL: string = 'CC-MAIN-2024-33'; // August 2024 index
+
+  public async fetchIndexes(): Promise<IndexEntryRaw[]> {
+    const res: Response = await fetch(CommonCrawlService.COMMON_CRAWL_URL + 'collinfo.json');
+    const output: IndexEntryRaw[] = await res.json();
+    return output;
+  }
+
+  public async readPageData(entry: DomainIndexEntryRaw): Promise<any> {
+    const rval: Record<string, string[]> = {};
+    const langs: string[] = CommonCrawlService.validLanguages(entry);
+
+    for (const lang of langs) {
+      rval[lang] = [];
+      const data: WarcEntry = await this.pullPageEntry(entry);
+      const asString: string = data.content.toString();
+      const parsed: CheerioAPI = cheerio.load(asString);
+      ['p', 'div', 'span'].forEach((tag) => {
+        parsed(tag).each((idx: number, el) => {
+          const txt: string = StringRatchet.trimToNull(parsed(el).text());
+          if (txt && txt.includes('.')) {
+            rval[lang].push(txt);
+            //Logger.info('Div: %s : type %s : text %s', idx, el.type, txt);
+          }
+        });
+      });
+    }
+    return rval;
+  }
+
+  public static validLanguages(entry: DomainIndexEntryRaw): string[] {
+    const validLangs: string[] = entry.languages
+      .split(',')
+      .map((s) => StringRatchet.trimToNull(s))
+      .filter((s) => !!s);
+    return validLangs;
+  }
+
+  public async pullPageEntry(entry: DomainIndexEntryRaw, language?: string): Promise<WarcEntry> {
+    const prefix: string = 'https://data.commoncrawl.org/'; //https://aws-publicdatasets.s3.amazonaws.com/';
+    const url: string = prefix + entry.filename;
+
+    const headers: Record<string, string> = { Range: 'bytes=' + entry.offset + '-' + (entry.offset + entry.length + 1) };
+    if (language) {
+      if (!CommonCrawlService.validLanguages(entry).includes(language)) {
+        throw ErrorRatchet.fErr('Requested language %s, but valid are %s', language, entry.languages);
+      }
+      headers['Accept-Language'] = language;
+    }
+
+    const resp: Response = await fetch(url, {
+      headers: headers,
+    });
+    //const do_unzip = promisify(unzip);
+
+    let reader: Readable = null;
+    if (resp.body instanceof Readable) {
+      reader = resp.body;
+    } else if (resp.body instanceof ReadableStream) {
+      reader = NodeStreamRatchet.webReadableStreamToNodeReadable(resp.body);
+    }
+
+    const warcstream: warc = new warc();
+    //const unzip: zlib.Gunzip = zlib.createGunzip();
+    //const unzipped: Buffer = zlib.unzipSync(await resp.arrayBuffer());
+
+    //const body: Readable = resp.body as unknown as Readable; // as unknown as PassThrough;
+    //const nodeReadble: Readable = NodeStreamRatchet.webReadableStreamToNodeReadable(resp.body);
+
+    Logger.info('Headers is %j', resp.headers);
+
+    let rval: WarcEntryRaw = null;
+    //let done: boolean = false;
+    reader
+      .pipe(zlib.createGunzip())
+      .pipe(warcstream)
+      .on('data', (val: WarcEntryRaw) => {
+        Logger.info('Got data ' + val.content.length);
+        rval = val; //.push(val);
+        //if (val.headers['WARC-Target-URI'] === entry.url) {
+        // }
+        //rval = val;
+        warcstream.destroy();
+      })
+      .on('close', () => {
+        Logger.info('Got close event');
+        //done = true;
+      })
+      .on('error', (err) => {
+        Logger.error('Read error: %s', err, err);
+        //done = true;
+      });
+
+    while (!rval) {
+      // Shouldnt really happen if offset is correct
+      await PromiseRatchet.wait(500);
+    }
+
+    const conv: WarcEntry = {
+      protocol: rval.protocol,
+      headers: rval.headers,
+      content: rval.content.toString(),
+    };
+
+    return conv;
+  }
+
+  public async search(options: CommonCrawlFetchOptions): Promise<DomainIndexEntryRaw[]> {
+    RequireRatchet.notNullOrUndefined(options, 'options');
+    RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(options.url, 'options.url');
+
+    let url: string = CommonCrawlService.COMMON_CRAWL_URL + (options.index || CommonCrawlService.CURRENT_CRAWL);
+    url += '-index?';
+    const params = {
+      url: options.url,
+      //from: options.from,
+      //to: options.to,
+      matchType: options.matchType || 'domain', // exact, prefix, host , domain,
+      //limit: options.limit,
+      //sort: options.sort,
+      //page: options.page,
+      //pageSize: options.pageSize,
+      //showNumPages: options.showNumPages || false,
+      output: 'json',
+    };
+
+    const urlPart: string = querystring.stringify(params);
+    url += urlPart;
+
+    //Logger.info('URL: %s', url);
+
+    const res: Response = await fetch(url);
+    const body: string = await res.text();
+    let rval: DomainIndexEntryRaw[] = null;
+    if (res.status === 200) {
+      const lines: string[] = body.split('\n');
+      rval = lines.map((s) => (StringRatchet.trimToNull(s) ? JSON.parse(s) : null)).filter((s) => !!s);
+      //Logger.info('%j', rval);
+    } else {
+      Logger.error('Failed to fetch: %s : %s : %j : %s', res.status, res.statusText, res.headers, res.body);
+    }
+
+    return rval;
+  }
+
+  /*
+  public static async gunzip(input: ReadableStream): Promise<Buffer> {
+    const promise = new Promise<Buffer>(function (resolve, reject) {
+      zlib.gunzip(input, function (error, result) {
+        if (!error) resolve(result);
+        else reject(error);
+      });
+    });
+    return promise;
+  }
+
+   */
+
+  public async scanSite(
+    opts: CommonCrawlFetchOptions,
+    onPage?: (idx: number, cnt: number, header: string) => Promise<any>,
+  ): Promise<CommonCrawlScan> {
+    const sw: StopWatch = new StopWatch();
+    const rval: CommonCrawlScan = {
+      options: opts,
+      pageIndexes: [],
+      parsed: [],
+      errors: [],
+    };
+    Logger.info('Performing domain index scan with %j', opts);
+    rval.pageIndexes = await this.search(opts);
+    Logger.info('Found %d entries, pulling each', rval.pageIndexes.length);
+    for (const [idx, ent] of rval.pageIndexes.entries()) {
+      try {
+        Logger.info('Pulling item %d of %d, %s', idx, rval.pageIndexes.length, sw.dumpExpected(idx / rval.pageIndexes.length));
+        if (onPage) {
+          try {
+            await onPage(idx, rval.pageIndexes.length, ent.url);
+          } catch (err) {
+            Logger.warn('Failed onpage: %s', err);
+          }
+        }
+        const parsed: WarcEntry = await this.pullPageEntry(ent);
+
+        rval.parsed.push(parsed);
+      } catch (err) {
+        Logger.warn('Failed to pull %j : %s', ent, err);
+        rval.errors.push({ pageIdx: ent, error: err });
+      }
+    }
+    Logger.info('Completed full scan in %s', sw.dump());
+    return rval;
+  }
+}