@mherod/get-cookie 2.0.0-rc.1 → 2.0.0-rc.11

package/src/cli.ts

@@ -1,118 +1,126 @@
 #!/usr/bin/env node
 
-import minimist from "minimist";
-import { argv } from "./argv";
-import { env } from "./global";
-import { queryCookies } from "./queryCookies";
+import { argv, parsedArgs } from "./argv";
 import { groupBy } from "lodash";
-import { green, yellow } from "colorette";
+import { green, red, yellow } from "colorette";
 import { resultsRendered } from "./resultsRendered";
 import { fetchWithCookies } from "./fetchWithCookies";
 import { unpackHeaders } from "./unpackHeaders";
+import CookieSpec from "./CookieSpec";
+import { comboQueryCookieSpec } from "./comboQueryCookieSpec";
+import { cookieSpecsFromUrl } from "./cookieSpecsFromUrl";
 
-const parsedArgs: minimist.ParsedArgs = minimist(argv);
-
-async function cliQueryCookies(name: string, domain: string) {
+async function cliQueryCookies(cookieSpec: CookieSpec | CookieSpec[]) {
   try {
-    const results = await queryCookies({ name, domain });
-    if (results.length > 0) {
-      if (argv.includes("--combined-string")) {
-        console.log(yellow(resultsRendered(results)));
-      } else if (argv.includes("--render") || argv.includes("-r")) {
-        console.log(yellow(resultsRendered(results)));
-      } else if (argv.includes("--dump") || argv.includes("-d")) {
-        console.log(results);
-      } else if (argv.includes("--dump-grouped")) {
-        const groupedByFile = groupBy(results, (r) => r.meta?.file);
-        console.log(green(JSON.stringify(groupedByFile, null, 2)));
-      } else if (argv.includes("--combined-string-grouped")) {
-        const groupedByFile = groupBy(results, (r) => r.meta?.file);
-        for (const file of Object.keys(groupedByFile)) {
-          let results = groupedByFile[file];
-          console.log(green(file) + ": ", yellow(resultsRendered(results)));
-        }
-      } else {
-        for (const result of results) {
-          console.log(result.value);
-        }
+    const results = await comboQueryCookieSpec(cookieSpec);
+    if (results == null || results.length == 0) {
+      console.error(red("No results"));
+      return;
+    }
+    if (parsedArgs["dump"] || parsedArgs["d"]) {
+      console.log(results);
+      return;
+    }
+    if (parsedArgs["dump-grouped"] || parsedArgs["D"]) {
+      const groupedByFile = groupBy(results, (r) => r.meta?.file);
+      console.log(green(JSON.stringify(groupedByFile, null, 2)));
+      return;
+    }
+    if (
+      parsedArgs["render"] ||
+      parsedArgs["render-merged"] ||
+      parsedArgs["r"]
+    ) {
+      console.log(yellow(resultsRendered(results)));
+      return;
+    }
+    if (parsedArgs["render-grouped"] || parsedArgs["R"]) {
+      const groupedByFile = groupBy(results, (r) => r.meta?.file);
+      for (const file of Object.keys(groupedByFile)) {
+        let results = groupedByFile[file];
+        console.log(green(file) + ": ", yellow(resultsRendered(results)));
       }
-    } else {
-      console.error("No results");
+      return;
+    }
+    for (const result of results) {
+      console.log(result.value);
     }
   } catch (e) {
     console.error(e);
   }
 }
 
-function main() {
-  if (argv && argv.length > 2) {
-    const arg2 = argv[2];
-    const arg3 = argv[3];
-    if (arg2 == "--fetch" || arg2 == "-f") {
-      const f = parsedArgs["f"];
-      let url: URL;
-      try {
-        url = new URL(f);
-      } catch (e) {
-        console.error("Invalid URL", arg3);
-        return;
+async function main() {
+  if (parsedArgs["help"] || parsedArgs["h"]) {
+    console.log(`Usage: ${argv[1]} [name] [domain] [options] `);
+    console.log(`Options:`);
+    console.log(`  -h, --help: Show this help`);
+    console.log(`  -v, --verbose: Show verbose output`);
+    console.log(`  -d, --dump: Dump all results`);
+    console.log(`  -D, --dump-grouped: Dump all results, grouped by profile`);
+    console.log(`  -r, --render: Render all results`);
+    return;
+  }
+
+  parsedArgs["verbose"] = parsedArgs["verbose"] || parsedArgs["v"];
+
+  const fetchUrl: string = parsedArgs["fetch"] || parsedArgs["F"];
+  if (fetchUrl) {
+    let url: URL;
+    try {
+      url = new URL(<string>fetchUrl);
+    } catch (e) {
+      console.error("Invalid URL", fetchUrl);
+      return;
+    }
+    const headerArgs: string[] | string = parsedArgs["H"];
+    const headers = unpackHeaders(headerArgs);
+    const onfulfilled = (res: Response) => {
+      if (parsedArgs["dump-response-headers"]) {
+        res.headers.forEach((value: string, key: string) => {
+          console.log(`${key}: ${value}`);
+        });
       }
-      const headerArgs: string[] | string = parsedArgs["H"];
-      const headers = unpackHeaders(headerArgs);
-      const onfulfilled = (res: Response) => {
-        return res.text().then((r) => {
+      if (parsedArgs["dump-response-body"]) {
+        res.text().then((r) => {
           console.log(r);
         });
-      };
-      fetchWithCookies(
-        url,
-        {
-          //
-          headers,
-        }
-        //
-      ).then(
-        onfulfilled,
-        console.error
-        //
-      );
+      }
       return;
-    }
-    let domain;
-    if (arg3 != null && arg3.indexOf(".") > -1) {
-      domain = arg3;
-    } else {
-      domain = "%";
-    }
-
-    const tru = `${true}`;
-
-    if (argv.includes("--require-jwt")) {
-      env.REQUIRE_JWT = tru;
-    }
-    if (argv.includes("--verbose")) {
-      env.VERBOSE = tru;
-    }
-    if (argv.includes("--chrome-only")) {
-      env.CHROME_ONLY = tru;
-    }
-    if (argv.includes("--firefox-only")) {
-      env.FIREFOX_ONLY = tru;
-    }
-    if (argv.includes("--ignore-expired")) {
-      env.IGNORE_EXPIRED = tru;
-    }
-
-    if (argv.includes("--single")) {
-      env.SINGLE = tru;
-    }
+    };
+    fetchWithCookies(
+      url,
+      {
+        //
+        headers,
+      }
+      //
+    ).then(
+      onfulfilled,
+      console.error
+      //
+    );
+    return;
+  }
 
-    if (env.VERBOSE) {
-      console.log("Verbose mode", argv);
+  const cookieSpecs: CookieSpec[] = [];
+  const argUrl: string = parsedArgs["url"] || parsedArgs["u"];
+  if (argUrl) {
+    for (const cookieSpec of cookieSpecsFromUrl(argUrl)) {
+      cookieSpecs.push(cookieSpec);
     }
+  } else {
+    cookieSpecs.push({
+      name: parsedArgs["name"] || parsedArgs["_"][0] || "%",
+      domain: parsedArgs["domain"] || parsedArgs["_"][1] || "%",
+    });
+  }
 
-    cliQueryCookies(arg2, domain).catch(console.error);
+  if (parsedArgs.verbose) {
+    console.log("cookieSpecs", cookieSpecs);
   }
+
+  await cliQueryCookies(cookieSpecs).catch(console.error);
 }
 
-main();
+main().then((r) => r, console.error);

package/src/comboQueryCookieSpec.ts

@@ -0,0 +1,21 @@
+import CookieSpec, { MultiCookieSpec } from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import { queryCookies } from "./queryCookies";
+import { uniqBy } from "lodash";
+
+export async function comboQueryCookieSpec(
+  cookieSpec: MultiCookieSpec
+): Promise<ExportedCookie[]> {
+  const cookies: ExportedCookie[] = [];
+  if (Array.isArray(cookieSpec)) {
+    const cookieSpecs = <CookieSpec[]>cookieSpec;
+    for (const cookieSpec1 of cookieSpecs) {
+      const cookies1: ExportedCookie[] = await queryCookies(cookieSpec1);
+      cookies.push(...cookies1);
+    }
+  } else {
+    const singleQuery = await queryCookies(cookieSpec);
+    cookies.push(...singleQuery);
+  }
+  return uniqBy(cookies, JSON.stringify);
+}

package/src/cookieSpecsFromUrl.ts

@@ -0,0 +1,26 @@
+import CookieSpec from "./CookieSpec";
+import { uniqBy } from "lodash";
+
+export function cookieSpecsFromUrl(url: URL | string): CookieSpec[] {
+  const url1 = typeof url == "string" ? new URL(url) : url;
+  const cookieSpecs = [];
+  const splits = url1.hostname.split(".");
+  const tld = splits.slice(-2).join(".");
+  const cookieSpec: CookieSpec = {
+    name: "%",
+    domain: "%." + tld,
+  };
+  cookieSpecs.push(cookieSpec);
+  const cookieSpec1: CookieSpec = {
+    name: "%",
+    domain: url1.hostname,
+  };
+  cookieSpecs.push(cookieSpec1);
+  // const isWww = splits.slice(-3)[0] === "www";
+  const cookieSpec2: CookieSpec = {
+    name: "%",
+    domain: tld,
+  };
+  cookieSpecs.push(cookieSpec2);
+  return uniqBy(cookieSpecs, JSON.stringify);
+}

package/src/doSqliteQuery1.ts

@@ -1,13 +1,9 @@
 import * as fs from "fs";
 import * as sqlite3 from "sqlite3";
-import { CookieRow } from "./CookieRow";
+import CookieRow from "./CookieRow";
 import { merge } from "lodash";
-
-interface DoSqliteQuery1Params {
-  file: string;
-  sql: string;
-  rowTransform: (row: any) => CookieRow;
-}
+import { parsedArgs } from "./argv";
+import { DoSqliteQuery1Params } from "./doSqliteQuery1Params";
 
 export async function doSqliteQuery1({
   file,
@@ -42,7 +38,7 @@ export async function doSqliteQuery1({
         resolve(cookieRows);
         return;
       }
-      if (process.env.VERBOSE) {
+      if (parsedArgs.verbose) {
         console.log(`doSqliteQuery1: rows ${JSON.stringify(rows1)}`);
       }
       resolve([rows1]);

package/src/doSqliteQuery1Params.ts

@@ -0,0 +1,8 @@
+import CookieRow from "./CookieRow";
+
+export interface DoSqliteQuery1Params {
+  file: string;
+  sql: string;
+  rowFilter?: (row: any) => boolean;
+  rowTransform: (row: any) => CookieRow;
+}

package/src/fetchWithCookies.ts

@@ -1,63 +1,108 @@
 // noinspection JSUnusedGlobalSymbols
 
-import { fetch } from "cross-fetch";
+import { fetch as fetchImpl } from "cross-fetch";
 import { merge } from "lodash";
 // noinspection SpellCheckingInspection
 import destr from "destr";
-import FetchResponse from "./FetchResponse";
-import { getGroupedRenderedCookies } from "./getGroupedRenderedCookies";
+import { cookieJar } from "./CookieStore";
+import UserAgent from "user-agents";
+import { parsedArgs } from "./argv";
+import { blue, yellow } from "colorette";
+import { getMergedRenderedCookies } from "./getMergedRenderedCookies";
+import { cookieSpecsFromUrl } from "./cookieSpecsFromUrl";
+import CookieSpec from "./CookieSpec";
 
 export async function fetchWithCookies(
   url: RequestInfo | URL,
-  options: RequestInit | undefined = {}
-): Promise<FetchResponse> {
+  options: RequestInit | undefined = {},
+  fetch: Function = fetchImpl
+): Promise<Response> {
+  const headers = {
+    "User-Agent": new UserAgent().toString(),
+  };
   const defaultOptions: RequestInit = {
-    headers: {
-      "User-Agent":
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-    },
+    headers,
     redirect: "manual",
   };
   const url2: string = `${url}`;
   const url1: URL = new URL(url2);
-  const domain = url1.hostname.replace(/^.*(\.\w+\.\w+)$/, (match, p1) => {
-    return `%${p1}`;
-  });
-  const cookies: string[] = await getGroupedRenderedCookies({
-    name: "%",
-    domain: domain,
-  });
-  const cookie = cookies.pop();
-  const newOptions1: RequestInit = merge(defaultOptions, options, {
-    headers: {
+  const cookieSpecs: CookieSpec[] = cookieSpecsFromUrl(url1);
+  const cookie = await getMergedRenderedCookies(cookieSpecs).catch(() => "");
+  if (cookie.length > 0) {
+    merge(headers, {
       Cookie: cookie,
-    },
-  });
+    });
+  }
+  if (parsedArgs["dump-request-headers"]) {
+    console.log(blue("Request headers:"), headers);
+  }
+  const newOptions1: RequestInit = merge(defaultOptions, options, { headers });
   try {
     const res: Response = await fetch(url2, newOptions1);
-    const newUrl = res.headers.get("location") as string;
+    const headers: [string, string][] = [];
+    res.headers.forEach((value, key) => {
+      headers.push([key, value]);
+    });
+    for (const [key, value] of headers) {
+      if (key === "set-cookie") {
+        await cookieJar.setCookie(value, url2);
+        if (parsedArgs.verbose) {
+          console.log(blue(`Set-Cookie: ${yellow(value)} ${yellow(url2)}`));
+        }
+        // const cookie = tough.parse(value);
+        // if (cookie instanceof Cookie) {
+        //   await memoryCookieStore.putCookie(cookie);
+        // }
+      }
+    }
+
+    const newUrl: string = res.headers.get("location") as string;
     if (res.redirected || (newUrl && newUrl !== url2)) {
+      if (parsedArgs.verbose) {
+        console.log(blue(`Redirected to `), yellow(newUrl));
+      }
       return fetchWithCookies(newUrl, newOptions1);
     }
-    const arrayBuffer1 = res.arrayBuffer();
-    const arrayBuffer = async () => arrayBuffer1;
-    const buffer = async () => arrayBuffer().then(Buffer.from);
-    const text = async () => buffer().then((buffer) => buffer.toString("utf8"));
-    const json = async () => text().then(destr);
-    const formData = async () =>
-      text().then((text) => new URLSearchParams(text));
+
+    const arrayBuffer1: Promise<ArrayBuffer> = res.arrayBuffer();
+
+    async function arrayBuffer(): Promise<ArrayBuffer> {
+      return arrayBuffer1;
+    }
+
+    async function buffer(): Promise<Buffer> {
+      return arrayBuffer().then(Buffer.from);
+    }
+
+    async function text(): Promise<string> {
+      return buffer().then((buffer) => buffer.toString("utf8"));
+    }
+
+    async function json(): Promise<any> {
+      return text().then((text) => destr(text));
+    }
+
+    async function formData(): Promise<FormData> {
+      const urlSearchParams: URLSearchParams = await text().then(
+        (text) => new URLSearchParams(text)
+      );
+      const formData = new FormData();
+      for (const [key, value] of urlSearchParams.entries()) {
+        formData.append(key, value);
+      }
+      return formData;
+    }
+
+    const res1: Response = res;
     const source2 = {
-      status: res.status,
-      statusText: res.statusText,
-      headers: res.headers,
       arrayBuffer,
-      buffer,
       text,
       json,
+      buffer,
       formData,
       //
     };
-    return merge({}, res, source2);
+    return merge(res1, source2);
   } catch (e) {
     throw e;
   }

package/src/findAllFiles.ts

@@ -1,5 +1,5 @@
-import { env } from "./global";
 import * as fs from "fs";
+import { parsedArgs } from "./argv";
 
 export async function findAllFiles({
   path,
@@ -12,16 +12,12 @@ export async function findAllFiles({
   maxDepth?: number;
 }): Promise<string[]> {
   const rootSegments = path.split("/").length;
-
-  if (env.VERBOSE) {
-    console.log(`Searching for ${name} in ${path}`);
-  }
   const files: string[] = [];
   let readdirSync;
   try {
     readdirSync = fs.readdirSync(path);
   } catch (e) {
-    if (env.VERBOSE) {
+    if (parsedArgs.verbose) {
       console.log(`Error reading ${path}`, e);
     }
     return [];
@@ -32,7 +28,7 @@ export async function findAllFiles({
     try {
       stat = fs.statSync(filePath);
     } catch (e) {
-      if (env.VERBOSE) {
+      if (parsedArgs.verbose) {
         console.error(`Error getting stat for ${filePath}`, e);
       }
       continue;
@@ -47,7 +43,7 @@ export async function findAllFiles({
           });
           files.push(...subFiles);
         } catch (e) {
-          if (env.VERBOSE) {
+          if (parsedArgs.verbose) {
             console.error(e);
           }
         }
@@ -56,7 +52,7 @@ export async function findAllFiles({
       files.push(filePath);
     }
   }
-  if (env.VERBOSE) {
+  if (parsedArgs.verbose) {
     if (files.length > 0) {
       console.log(`Found ${files.length} ${name} files`);
       console.log(files);

package/src/getChromeCookie.ts

@@ -0,0 +1,19 @@
+import CookieSpec from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import { queryCookies } from "./queryCookies";
+import ChromeCookieQueryStrategy from "./browsers/ChromeCookieQueryStrategy";
+import { isExportedCookie } from "./IsExportedCookie";
+
+export async function getChromeCookie(
+  params: CookieSpec
+): Promise<ExportedCookie | undefined> {
+  const cookies = await queryCookies(
+    params,
+    new ChromeCookieQueryStrategy()
+    //
+  );
+  if (cookies.length == 0) {
+    throw new Error("Cookie not found");
+  }
+  return cookies.find(isExportedCookie);
+}

package/src/getCookie.ts

@@ -0,0 +1,19 @@
+import CookieSpec from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import { queryCookies } from "./queryCookies";
+import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
+
+export async function getCookie(
+  params: CookieSpec
+): Promise<ExportedCookie | undefined> {
+  const cookies = await queryCookies(
+    params,
+    new CompositeCookieQueryStrategy()
+    //
+  );
+  if (Array.isArray(cookies) && cookies.length > 0) {
+    return cookies.find((cookie) => cookie != null);
+  } else {
+    throw new Error("Cookie not found");
+  }
+}

package/src/getFirefoxCookie.ts

@@ -0,0 +1,19 @@
+import CookieSpec from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import { queryCookies } from "./queryCookies";
+import FirefoxCookieQueryStrategy from "./browsers/FirefoxCookieQueryStrategy";
+
+export async function getFirefoxCookie(
+  params: CookieSpec
+): Promise<ExportedCookie | undefined> {
+  const cookies = await queryCookies(
+    params,
+    new FirefoxCookieQueryStrategy()
+    //
+  );
+  if (Array.isArray(cookies) && cookies.length > 0) {
+    return cookies.find((cookie) => cookie != null);
+  } else {
+    throw new Error("Cookie not found");
+  }
+}

package/src/getGroupedRenderedCookies.ts

@@ -1,32 +1,19 @@
-import { queryCookies } from "./queryCookies";
-import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
 import { groupBy } from "lodash";
 import { resultsRendered } from "./resultsRendered";
-import { CookieSpec } from "./CookieSpec";
-import { ExportedCookie } from "./ExportedCookie";
+import { MultiCookieSpec } from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import { comboQueryCookieSpec } from "./comboQueryCookieSpec";
 
 export async function getGroupedRenderedCookies(
-  //
-  {
-    name,
-    domain,
-  }: //
-  CookieSpec
-): //
-Promise<string[]> {
-  const cookies: ExportedCookie[] = await queryCookies(
-    { name, domain },
-    new CompositeCookieQueryStrategy()
-    //
-  );
-  if (Array.isArray(cookies) && cookies.length > 0) {
-    const results: ExportedCookie[] = await queryCookies({ name, domain });
-    const groupedByFile = groupBy(results, (r) => r.meta?.file);
-    return Object.keys(groupedByFile).map((file: string) => {
-      const results: ExportedCookie[] = groupedByFile[file];
-      return resultsRendered(results);
-    });
-  } else {
+  cookieSpec: MultiCookieSpec
+): Promise<string[]> {
+  const cookies: ExportedCookie[] = await comboQueryCookieSpec(cookieSpec);
+  if (cookies.length == 0) {
     throw new Error("Cookie not found");
   }
+  const groupedByFile = groupBy(cookies, (r: ExportedCookie) => r.meta?.file);
+  return Object.keys(groupedByFile).map((file: string) => {
+    const results: ExportedCookie[] = groupedByFile[file];
+    return resultsRendered(results);
+  });
 }

package/src/getMergedRenderedCookies.ts

@@ -0,0 +1,14 @@
+import { resultsRendered } from "./resultsRendered";
+import { comboQueryCookieSpec } from "./comboQueryCookieSpec";
+import { MultiCookieSpec } from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+
+export async function getMergedRenderedCookies(
+  cookieSpec: MultiCookieSpec
+): Promise<string> {
+  const cookies: ExportedCookie[] = await comboQueryCookieSpec(cookieSpec);
+  if (cookies.length > 0) {
+    return resultsRendered(cookies);
+  }
+  return "";
+}