@mherod/get-cookie 2.0.0-rc.8 → 2.1.0

package/src/fetchWithCookies.ts

@@ -1,93 +1,170 @@
-// noinspection JSUnusedGlobalSymbols
+// noinspection JSUnusedGlobalSymbols,ExceptionCaughtLocallyJS
 
 import { fetch as fetchImpl } from "cross-fetch";
 import { merge } from "lodash";
 // noinspection SpellCheckingInspection
 import destr from "destr";
-import CookieSpec from "./CookieSpec";
-import { getGroupedRenderedCookies } from "./getGroupedRenderedCookies";
-import { cookieJar } from "./CookieStore";
-import UserAgent from "user-agents";
 import { parsedArgs } from "./argv";
-import { blue, yellow } from "colorette";
+import { getMergedRenderedCookies } from "./getMergedRenderedCookies";
+import { cookieSpecsFromUrl } from "./cookieSpecsFromUrl";
+import CookieSpec from "./CookieSpec";
+import consola from "consola";
+
+if (typeof fetchImpl !== "function") {
+  throw new Error("fetch is not a function");
+}
+
+function constructUserAgent(
+  platform = "Macintosh; Intel Mac OS X 10_15_7",
+  engine = "AppleWebKit/537.36 (KHTML, like Gecko)",
+  browser = "Chrome/118.0.0.0",
+  layout = "Safari/537.36",
+): string {
+  return [platform, engine, browser, layout].join(" ");
+}
+
+const userAgent = constructUserAgent();
+
+interface FetchRequestInit {
+  url: RequestInfo | URL | string;
+  options?: RequestInit;
+}
+
+export type FetchFn =
+  | typeof fetchImpl
+  | ((url: URL, options?: RequestInit) => Promise<Response>);
 
 export async function fetchWithCookies(
-  url: RequestInfo | URL,
+  url: RequestInfo | URL | string,
   options: RequestInit | undefined = {},
-  fetch: Function = fetchImpl
+  fetch: FetchFn = fetchImpl as FetchFn,
+  originalRequest?: FetchRequestInit,
 ): Promise<Response> {
+  if (typeof fetch !== "function") {
+    const message = "fetch is not a function";
+    consola.error(message);
+    throw new Error(message);
+  }
+  const originalRequest1 = originalRequest || { url, options };
+  const headers: HeadersInit = {
+    "User-Agent": userAgent,
+  };
   const defaultOptions: RequestInit = {
-    headers: {
-      "User-Agent": new UserAgent().toString(),
-    },
+    headers,
     redirect: "manual",
   };
   const url2: string = `${url}`;
   const url1: URL = new URL(url2);
-  const domain = url1.hostname.replace(/^.*(\.\w+\.\w+)$/, (match, p1) => {
-    return `%${p1}`;
-  });
-  const cookieSpec: CookieSpec = {
-    name: "%",
-    domain: domain,
-  };
-  const cookies: string[] = await getGroupedRenderedCookies(cookieSpec).catch(
-    () => []
-  );
-  const cookie = cookies.pop();
-  const newOptions1: RequestInit = merge(defaultOptions, options, {
-    headers: {
-      Cookie: cookie,
+  consola.start("fetchWithCookies", url2);
+  const cookieSpecs: CookieSpec[] = cookieSpecsFromUrl(url1);
+  const renderedCookie = await getMergedRenderedCookies(cookieSpecs).catch(
+    (err) => {
+      consola.error(err);
+      return "";
     },
-  });
+  );
+  if (renderedCookie) {
+    headers["Cookie"] = renderedCookie;
+  } else {
+    consola.info("No cookies found for this request");
+  }
+  if (parsedArgs["dump-request-headers"]) {
+    consola.info("Request URL:", url1.href);
+    consola.info("Request headers:", headers);
+  }
+  const newOptions1: RequestInit = merge(defaultOptions, { headers }, options);
   try {
-    const res: Response = await fetch(url2, newOptions1);
+    const res: Response = await fetch(url1, newOptions1);
+    // noinspection JSMismatchedCollectionQueryUpdate
     const headers: [string, string][] = [];
     res.headers.forEach((value, key) => {
       headers.push([key, value]);
     });
-    for (const [key, value] of headers) {
-      if (key === "set-cookie") {
-        await cookieJar.setCookie(value, url2);
-        if (parsedArgs.verbose) {
-          console.log(blue(`Set-Cookie: ${yellow(value)} ${yellow(url2)}`));
+    // for (const [key, value] of headers) {
+    //   if (key === "set-cookie") {
+    //     const cookieJar1 = await cookieJarPromise;
+    //     await cookieJar1.setCookie(value, url2);
+    //     if (parsedArgs.verbose) {
+    //       console.log(blue(`Set-Cookie:`), yellow(value), yellow(url2));
+    //     }
+    //   }
+    // }
+
+    const newUrl: string = res.headers.get("location") ?? res.url;
+    // const sameHost = new URL(newUrl).host === url1.host;
+    if (res.status == 301 || res.status == 302) {
+      // follow the redirect
+      if (newUrl && newUrl !== url2) {
+        if (parsedArgs.verbose || parsedArgs["dump-response-headers"]) {
+          consola.info(`Redirected to `, newUrl);
         }
-        // const cookie = tough.parse(value);
-        // if (cookie instanceof Cookie) {
-        //   await memoryCookieStore.putCookie(cookie);
-        // }
+        return fetchWithCookies(
+          //
+          newUrl,
+          newOptions1,
+          fetch,
+          originalRequest1,
+          //
+        );
       }
     }
-
-    const newUrl: string = res.headers.get("location") as string;
-    if (res.redirected || (newUrl && newUrl !== url2)) {
+    if (res.status == 303 && newUrl && newUrl !== url2) {
+      // follow the redirect with GET
+      let newOptions2: RequestInit = {};
+      switch (newOptions1.method) {
+        case "POST":
+        case "PUT":
+        case "DELETE":
+          merge(newOptions2, newOptions1, { method: "GET" });
+          newOptions2.body = undefined; // TODO: is this needed?
+          break;
+        case "HEAD":
+        case "GET":
+          merge(newOptions2, newOptions1);
+          newOptions2.body = undefined; // TODO: is this needed?
+          break;
+        default:
+          merge(newOptions2, newOptions1);
+          break;
+      }
       if (parsedArgs.verbose) {
-        console.log(blue(`Redirected to `), yellow(newUrl));
+        console.log(`Redirected to `, newUrl);
       }
-      return fetchWithCookies(newUrl, newOptions1);
+      return fetchWithCookies(
+        //
+        newUrl,
+        newOptions2,
+        fetch,
+        originalRequest1,
+        //
+      );
     }
 
-    const arrayBuffer1: Promise<ArrayBuffer> = res.arrayBuffer();
+    // all deserialize functions will use the arraybuffer
+    const arrayBufferPromise: Promise<ArrayBuffer> = res.arrayBuffer();
+    const bufferPromise = arrayBufferPromise.then(Buffer.from);
 
     async function arrayBuffer(): Promise<ArrayBuffer> {
-      return arrayBuffer1;
+      return arrayBufferPromise;
     }
 
     async function buffer(): Promise<Buffer> {
-      return arrayBuffer().then(Buffer.from);
+      return bufferPromise;
     }
 
     async function text(): Promise<string> {
-      return buffer().then((buffer) => buffer.toString("utf8"));
+      const buffer1 = await bufferPromise;
+      return buffer1.toString("utf8");
     }
 
     async function json(): Promise<any> {
-      return text().then((text) => destr(text));
+      const text1 = await text();
+      return destr(text1);
     }
 
     async function formData(): Promise<FormData> {
       const urlSearchParams: URLSearchParams = await text().then(
-        (text) => new URLSearchParams(text)
+        (text) => new URLSearchParams(text),
       );
       const formData = new FormData();
       for (const [key, value] of urlSearchParams.entries()) {

package/src/findAllFiles.ts

@@ -1,62 +1,38 @@
-import * as fs from "fs";
+import { existsSync } from "fs";
 import { parsedArgs } from "./argv";
+import consola from "consola";
+import { sync } from "fast-glob";
 
-export async function findAllFiles({
-  path,
-  name,
-  maxDepth = 2,
-}: //
-{
+type FindFilesOptions = {
   path: string;
   name: string;
   maxDepth?: number;
-}): Promise<string[]> {
-  const rootSegments = path.split("/").length;
-  const files: string[] = [];
-  let readdirSync;
-  try {
-    readdirSync = fs.readdirSync(path);
-  } catch (e) {
-    if (parsedArgs.verbose) {
-      console.log(`Error reading ${path}`, e);
-    }
-    return [];
+};
+
+export function findAllFiles({
+  path,
+  name,
+  maxDepth = 2,
+}: FindFilesOptions): string[] {
+  if (!existsSync(path)) {
+    throw new Error(`Path ${path} does not exist`);
   }
-  for (const file of readdirSync) {
-    const filePath = path + "/" + file;
-    let stat;
-    try {
-      stat = fs.statSync(filePath);
-    } catch (e) {
-      if (parsedArgs.verbose) {
-        console.error(`Error getting stat for ${filePath}`, e);
-      }
-      continue;
-    }
-    if (stat.isDirectory()) {
-      if (filePath.split("/").length < rootSegments + maxDepth) {
-        try {
-          const subFiles = await findAllFiles({
-            path: filePath,
-            name: name,
-            maxDepth: 2,
-          });
-          files.push(...subFiles);
-        } catch (e) {
-          if (parsedArgs.verbose) {
-            console.error(e);
-          }
-        }
-      }
-    } else if (file === name) {
-      files.push(filePath);
-    }
+
+  if (parsedArgs.verbose) {
+    consola.start(`Searching for ${name} files in ${path}`);
   }
+
+  const files: string[] = sync(`${path}/**/${name}`, {
+    onlyFiles: true,
+    deep: maxDepth,
+  });
+
   if (parsedArgs.verbose) {
     if (files.length > 0) {
-      console.log(`Found ${files.length} ${name} files`);
-      console.log(files);
+      consola.success(`Found ${files.length} ${name} files`);
+      consola.info(files);
     }
   }
+
   return files;
 }

package/src/getChromeCookie.ts

@@ -1,15 +1,17 @@
 import CookieSpec from "./CookieSpec";
 import ExportedCookie from "./ExportedCookie";
 import { queryCookies } from "./queryCookies";
-import ChromeCookieQueryStrategy from "./browsers/ChromeCookieQueryStrategy";
-import { isExportedCookie } from "./IsExportedCookie";
+import ChromeCookieQueryStrategy from "./browsers/chrome/ChromeCookieQueryStrategy";
+import { isExportedCookie } from "./ExportedCookie";
 
 export async function getChromeCookie(
-  params: CookieSpec
+  params: CookieSpec,
 ): Promise<ExportedCookie | undefined> {
   const cookies = await queryCookies(
     params,
-    new ChromeCookieQueryStrategy()
+    {
+      strategy: new ChromeCookieQueryStrategy(),
+    },
     //
   );
   if (cookies.length == 0) {

package/src/getCookie.ts

@@ -4,11 +4,14 @@ import { queryCookies } from "./queryCookies";
 import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
 
 export async function getCookie(
-  params: CookieSpec
+  params: CookieSpec,
 ): Promise<ExportedCookie | undefined> {
-  const cookies = await queryCookies(
+  //
+  const cookies: ExportedCookie[] = await queryCookies(
     params,
-    new CompositeCookieQueryStrategy()
+    {
+      strategy: new CompositeCookieQueryStrategy(),
+    },
     //
   );
   if (Array.isArray(cookies) && cookies.length > 0) {

package/src/getFirefoxCookie.ts

@@ -1,14 +1,16 @@
 import CookieSpec from "./CookieSpec";
 import ExportedCookie from "./ExportedCookie";
 import { queryCookies } from "./queryCookies";
-import FirefoxCookieQueryStrategy from "./browsers/FirefoxCookieQueryStrategy";
+import FirefoxCookieQueryStrategy from "./browsers/firefox/FirefoxCookieQueryStrategy";
 
 export async function getFirefoxCookie(
-  params: CookieSpec
+  params: CookieSpec,
 ): Promise<ExportedCookie | undefined> {
-  const cookies = await queryCookies(
+  const cookies: ExportedCookie[] = await queryCookies(
     params,
-    new FirefoxCookieQueryStrategy()
+    {
+      strategy: new FirefoxCookieQueryStrategy(),
+    },
     //
   );
   if (Array.isArray(cookies) && cookies.length > 0) {

package/src/getGroupedRenderedCookies.ts

@@ -1,27 +1,17 @@
 import { groupBy } from "lodash";
-import { queryCookies } from "./queryCookies";
 import { resultsRendered } from "./resultsRendered";
-import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
-import CookieSpec from "./CookieSpec";
+import { MultiCookieSpec } from "./CookieSpec";
 import ExportedCookie from "./ExportedCookie";
+import { comboQueryCookieSpec } from "./comboQueryCookieSpec";
 
-export async function getGroupedRenderedCookies({
-  name,
-  domain,
-}: CookieSpec): Promise<string[]> {
-  const cookies: ExportedCookie[] = await queryCookies(
-    {
-      name,
-      domain,
-    },
-    new CompositeCookieQueryStrategy()
-    //
-  );
+export async function getGroupedRenderedCookies(
+  cookieSpec: MultiCookieSpec,
+): Promise<string[]> {
+  const cookies: ExportedCookie[] = await comboQueryCookieSpec(cookieSpec);
   if (cookies.length == 0) {
     throw new Error("Cookie not found");
   }
-  const results: ExportedCookie[] = await queryCookies({ name, domain });
-  const groupedByFile = groupBy(results, (r: ExportedCookie) => r.meta?.file);
+  const groupedByFile = groupBy(cookies, (r: ExportedCookie) => r.meta?.file);
   return Object.keys(groupedByFile).map((file: string) => {
     const results: ExportedCookie[] = groupedByFile[file];
     return resultsRendered(results);

package/src/getMergedRenderedCookies.test.ts

@@ -0,0 +1,43 @@
+import { getMergedRenderedCookies } from "./getMergedRenderedCookies";
+import { MultiCookieSpec } from "./CookieSpec";
+import MockCookieQueryStrategy from "./browsers/mock/MockCookieQueryStrategy";
+
+describe("getMergedRenderedCookies", () => {
+  it("should return empty string if no cookies are present", async () => {
+    const cookieSpec: MultiCookieSpec = [];
+    const result = await getMergedRenderedCookies(cookieSpec);
+    expect(result).toBe("");
+  });
+
+  it("should return rendered cookies if cookies are present", async () => {
+    const cookieSpec: MultiCookieSpec = [
+      {
+        name: "sessionCookie",
+        domain: "www.example.com",
+      },
+      {
+        name: "persistentCookie",
+        domain: "www.example.com",
+      },
+    ];
+    const result = await getMergedRenderedCookies(
+      cookieSpec,
+      new MockCookieQueryStrategy([
+        {
+          name: "sessionCookie",
+          value: "sessionCookieValue",
+          domain: "www.example.com",
+        },
+        {
+          name: "persistentCookie",
+          value: "persistentCookieValue",
+          domain: "www.example.com",
+        },
+      ]),
+      //
+    );
+    expect(result).not.toBe("");
+    expect(result).toContain("sessionCookie=sessionCookieValue");
+    expect(result).toContain("persistentCookie=persistentCookieValue");
+  });
+});

package/src/getMergedRenderedCookies.ts

@@ -0,0 +1,17 @@
+import { resultsRendered } from "./resultsRendered";
+import { comboQueryCookieSpec } from "./comboQueryCookieSpec";
+import { MultiCookieSpec } from "./CookieSpec";
+import ExportedCookie from "./ExportedCookie";
+import consola from "consola";
+import CookieQueryStrategy from "./browsers/CookieQueryStrategy";
+import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
+
+export async function getMergedRenderedCookies(
+  cookieSpec: MultiCookieSpec,
+  strategy: CookieQueryStrategy = new CompositeCookieQueryStrategy(),
+): Promise<string> {
+  const cookies: ExportedCookie[] = await comboQueryCookieSpec(cookieSpec, {
+    strategy,
+  });
+  return cookies.length > 0 ? resultsRendered(cookies) : "";
+}

package/src/global.ts

@@ -1,7 +1,7 @@
 import { merge } from "lodash";
 
 export const env: any = {};
-merge(env, process.env);
+merge(env, process?.env ?? {});
 export const HOME: string = env["HOME"];
 if (!HOME) {
   throw new Error("HOME environment variable is not set");

package/src/index.ts

@@ -1,21 +1,16 @@
-#!/usr/bin/env node
-// noinspection JSUnusedGlobalSymbols
-
 import { getCookie } from "./getCookie";
 import { getChromeCookie } from "./getChromeCookie";
 import { getFirefoxCookie } from "./getFirefoxCookie";
 import { getGroupedRenderedCookies } from "./getGroupedRenderedCookies";
+import { getMergedRenderedCookies } from "./getMergedRenderedCookies";
 import { fetchWithCookies } from "./fetchWithCookies";
 
 export {
   getCookie,
   getChromeCookie,
   getFirefoxCookie,
+  getMergedRenderedCookies,
   getGroupedRenderedCookies,
   fetchWithCookies,
   //
 };
-
-export * from "./CookieSpec";
-export * from "./CookieRow";
-export * from "./ExportedCookie";

package/src/isValidJwt.ts

@@ -1,11 +1,11 @@
 import jsonwebtoken, { JwtPayload } from "jsonwebtoken";
 import { parsedArgs } from "./argv";
 
-export default function isValidJwt(token: any) {
+export default function isValidJwt(token: string) {
   try {
     const result = jsonwebtoken.decode(token, { complete: true });
     if (parsedArgs.verbose && result) {
-      console.log(result);
+      console.debug(result);
     }
     const payload: JwtPayload = result?.payload as JwtPayload;
     if (payload) {

package/src/listChromeProfiles.ts

@@ -0,0 +1,45 @@
+import { sync } from "fast-glob";
+import { chromeApplicationSupport } from "./browsers/chrome/ChromeApplicationSupport";
+import { dirname } from "path";
+import { readFile } from "fs/promises";
+import { flatMapAsync } from "./util/flatMapAsync";
+import destr from "destr";
+
+export async function listChromeProfilePaths(): Promise<string[]> {
+  return sync(`./**/Cookies`, {
+    cwd: chromeApplicationSupport,
+    absolute: true,
+    onlyFiles: true,
+    deep: 2,
+  }).map((f) => {
+    // parent dir
+    return dirname(f);
+  });
+}
+
+export async function listChromeProfiles(): Promise<ChromeProfile[]> {
+  const paths = await listChromeProfilePaths();
+  return await flatMapAsync(paths, async (p) => {
+    const content = await readFile(`${p}/Preferences`, "utf-8");
+    return await destr(content);
+  });
+}
+
+export type ChromeProfileAccountInfo = {
+  account_id: string;
+  accountcapabilities: any;
+  email: string;
+  full_name: string;
+  gaia: string;
+  given_name: string;
+  hd: string;
+  is_supervised_child: number;
+  is_under_advanced_protection: boolean;
+  last_downloaded_image_url_with_size: string;
+  locale: string;
+  picture_url: string;
+};
+
+export type ChromeProfile = {
+  account_info: ChromeProfileAccountInfo[];
+};

package/src/logger.ts

@@ -0,0 +1,13 @@
+import { createConsola } from "consola";
+
+const consola = createConsola({
+  fancy: true,
+  formatOptions: {
+    showLogLevel: true,
+    colors: true,
+    date: false,
+  },
+});
+consola.wrapConsole();
+
+export default consola;

package/src/processBeforeReturn.ts

@@ -0,0 +1,26 @@
+import ExportedCookie from "./ExportedCookie";
+import { uniqBy } from "lodash";
+import CookieQueryStrategy from "./browsers/CookieQueryStrategy";
+import { CookieQueryOptions } from "./cookieQueryOptions";
+
+export function processBeforeReturn<T extends CookieQueryStrategy>(
+  cookies: ExportedCookie[],
+  options?: CookieQueryOptions<T>,
+): ExportedCookie[] {
+  if (options?.removeExpired) {
+    return cookies.filter((c: ExportedCookie) => {
+      const expiry: Date | "Infinity" | undefined = c.expiry;
+      if (expiry === undefined) {
+        return true;
+      } else if (expiry === "Infinity") {
+        return true;
+      } else {
+        return expiry.getTime() > Date.now();
+      }
+    });
+  }
+  if (options?.limit) {
+    return cookies.slice(0, options.limit);
+  }
+  return uniqBy(cookies, JSON.stringify);
+}

package/src/queryCookies.ts

@@ -3,13 +3,21 @@ import { parsedArgs } from "./argv";
 import CompositeCookieQueryStrategy from "./browsers/CompositeCookieQueryStrategy";
 import CookieQueryStrategy from "./browsers/CookieQueryStrategy";
 import isValidJwt from "./isValidJwt";
-import CookieSpec from "./CookieSpec";
+import CookieSpec, { MultiCookieSpec } from "./CookieSpec";
 import ExportedCookie from "./ExportedCookie";
+import { CookieQueryOptions } from "./cookieQueryOptions";
 
 export async function queryCookies(
-  { name, domain }: CookieSpec,
-  strategy: CookieQueryStrategy = new CompositeCookieQueryStrategy()
+  {
+    name,
+    domain,
+  }: //
+  CookieSpec,
+  options?: CookieQueryOptions<CookieQueryStrategy>,
 ): Promise<ExportedCookie[]> {
+  const strategy: CookieQueryStrategy =
+    options?.strategy || new CompositeCookieQueryStrategy();
+  //
   const results: ExportedCookie[] = await strategy.queryCookies(name, domain);
   const allCookies: ExportedCookie[] = uniqBy(results, JSON.stringify);