@liase/core 1.0.0

package/dist/ActionContext.js

@@ -0,0 +1,227 @@
+import { decodeHTML } from "entities";
+import { guessMediaInfoFromMimeType, guessMediaInfoFromUrl, } from "./actionHelpers.js";
+import { generateResponse, getResponseDetailsBasedOnRequest, } from "./generateResponse.js";
+import { headersToNormalisedBasicObject, parseFetchArgs } from "./lib/fetch.js";
+import { addCachingFetchWrapper } from "./lib/networkRequestsCache.js";
+import { loadUrl } from "./loadUrl.js";
+export const excludeFieldSymbol = Symbol("ExcludeField");
+export class ActionContext extends Function {
+    constructor(args) {
+        super();
+        this.#constructorContext = args.constructorContext;
+        this.#executeActions = args.executeActions;
+        this.#path = args.path;
+        if (args.initialData) {
+            this.#dataStore = args.initialData;
+        }
+        this.#networkRequestsHistory = args.networkRequestsHistory ?? [];
+        // biome-ignore lint/correctness/noConstructorReturn: Proxy wrapping requires returning from constructor
+        return new Proxy(this, {
+            apply: (target, thisArg, args) => target.get(...args),
+            get: (target, propName, receiver) => {
+                const value = target[propName];
+                if (value instanceof Function) {
+                    // biome-ignore lint/suspicious/noExplicitAny: spread args for dynamic proxy forwarding
+                    return (...args) => value.apply(this === receiver ? target : this, args);
+                }
+                return value;
+            },
+        });
+    }
+    #constructorContext;
+    #executeActions;
+    #path;
+    // biome-ignore lint/suspicious/noExplicitAny: result history stores arbitrary action results
+    #resultHistory = [];
+    #networkRequestsHistory;
+    // eslint-disable-next-line no-use-before-define -- we need to use before it's defined since it's recursive
+    #clonedChildren = [];
+    // biome-ignore lint/suspicious/noExplicitAny: unresolved promise store accepts arbitrary values
+    #unresolvedPromises = [];
+    // biome-ignore lint/suspicious/noExplicitAny: data store accepts arbitrary values
+    #dataStore = {};
+    get(key = "") {
+        if (!(key in this.#dataStore)) {
+            throw Error(`Attempted to access value "${key}" but that value was never set`);
+        }
+        return this.#dataStore[key];
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: value can be any type in dynamic data store
+    set(key, value) {
+        if (value instanceof Promise) {
+            this.#unresolvedPromises.push(value.then((resolvedValue) => {
+                if (this.get(key) === value) {
+                    this.set(key, resolvedValue);
+                }
+                else {
+                    throw Error(`The value saved under the key "${key}" was changed before the original value (which was a promise) finished resolving.`);
+                }
+            }));
+        }
+        this.#dataStore[key] = value;
+        return this;
+    }
+    has(key = "") {
+        return key in this.#dataStore;
+    }
+    getAll() {
+        return { ...this.#dataStore };
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: result can be any type from dynamic action execution
+    recordResult(result) {
+        this.#resultHistory.push(result);
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: result can be any type from dynamic action execution
+    lastResult() {
+        return this.#resultHistory[this.#resultHistory.length - 1];
+    }
+    clone({ path, appendToPath, data, } = {}) {
+        const clone = new ActionContext({
+            constructorContext: this.#constructorContext,
+            initialData: data ? { ...data } : { ...this.#dataStore },
+            executeActions: this.#executeActions,
+            path: (path ?? this.#path).concat(appendToPath ?? []),
+            networkRequestsHistory: this.#networkRequestsHistory,
+        });
+        this.#clonedChildren.push(clone);
+        return clone;
+    }
+    get descendants() {
+        return this.#clonedChildren.flatMap((clonedChild) => [
+            clonedChild,
+            ...clonedChild.descendants,
+        ]);
+    }
+    chain(...actions) {
+        return this.#executeActions(actions, this, this.#path);
+    }
+    waitForAllPromisesToResolve() {
+        return Promise.all(this.#unresolvedPromises);
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: request is a dynamic object from user input
+    get request() {
+        return Object.freeze(this.#constructorContext.request);
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: secrets is a dynamic object from user input
+    get secrets() {
+        return Object.freeze(this.#constructorContext.secrets);
+    }
+    get requestHandler() {
+        return Object.freeze(this.#constructorContext.requestHandler);
+    }
+    get pageFetchLimitReached() {
+        return this.#constructorContext.pageFetchLimitReached;
+    }
+    get cacheNetworkRequests() {
+        return this.#constructorContext.cacheNetworkRequests;
+    }
+    loadUrl = (async (url, options) => {
+        const response = (await loadUrl.call(this, url, 
+        // @ts-expect-error -- Not sure why ts doesn't like this being undefined
+        options));
+        let stringifiedBody;
+        if ("root" in response) {
+            stringifiedBody = response.root.nativeSelector.html() ?? "";
+        }
+        else if (typeof response.data === "string") {
+            stringifiedBody = response.data;
+        }
+        else {
+            stringifiedBody = JSON.stringify(response.data, null, 2);
+        }
+        this.#networkRequestsHistory.push({
+            constructorPath: this.#path,
+            request: {
+                url: new URL(url),
+                method: options?.method || "GET",
+                headers: response.request.headers,
+                body: options?.body,
+            },
+            response: {
+                headers: response.headers,
+                body: stringifiedBody,
+                statusCode: response.statusCode,
+                cached: response.cached,
+                cachedOn: response.cachedOn,
+            },
+        });
+        return response;
+        // biome-ignore lint/suspicious/noExplicitAny: cast needed for complex overloaded function assignment
+    });
+    get networkRequestsHistory() {
+        return this.#networkRequestsHistory;
+    }
+    loadRequest = async (requestHandler, request, { secrets = {}, } = {}) => {
+        const sourceId = request.source ?? this.#constructorContext.sourceId;
+        const fullRequest = {
+            ...request,
+            source: sourceId,
+            queryType: request.queryType ?? requestHandler.id,
+        };
+        const constructorContext = {
+            request: fullRequest,
+            secrets,
+            requestHandler,
+            responseDetails: getResponseDetailsBasedOnRequest(requestHandler.responses, fullRequest),
+            sourceId,
+            hooks: this.#constructorContext.hooks,
+        };
+        return generateResponse(constructorContext);
+    };
+    get hooks() {
+        return this.#constructorContext.hooks;
+    }
+    get fetch() {
+        const cachingFetch = addCachingFetchWrapper(fetch, this.#constructorContext.cacheNetworkRequests);
+        return async (input, init) => {
+            const response = await cachingFetch(input, init);
+            const { url, body, headers, method } = parseFetchArgs(input, init);
+            const clonedResponse = response.clone();
+            const cacheTimestamp = response.statusText.match(/^Cached on: (\d+)$/)?.[1];
+            const responseHeaders = headersToNormalisedBasicObject(clonedResponse.headers);
+            this.#networkRequestsHistory.push({
+                constructorPath: this.#path,
+                request: {
+                    url,
+                    method,
+                    headers,
+                    body,
+                },
+                response: {
+                    headers: responseHeaders,
+                    body: clonedResponse.text(),
+                    statusCode: clonedResponse.status,
+                    cached: Boolean(cacheTimestamp),
+                    cachedOn: cacheTimestamp
+                        ? new Date(Number.parseInt(cacheTimestamp))
+                        : null,
+                },
+            });
+            return response;
+        };
+    }
+    guessMediaInfoFromUrl = guessMediaInfoFromUrl;
+    guessMediaInfoFromMimeType = guessMediaInfoFromMimeType;
+    decodeHTML = (value) => decodeHTML(value);
+    durationStringToNumber = (duration) => {
+        const match = duration.match(/^\s*(?:(?:(\d+)[:D])?(\d{1,2})[:H])?(\d{1,2})[:M](\d{2})[S]?\s*$/);
+        if (!match) {
+            throw Error(`The value "${duration}" is not a valid duration string`);
+        }
+        const [, ...segments] = match;
+        let totalSeconds = 0;
+        if (typeof segments[0] === "string") {
+            totalSeconds += Number.parseInt(segments[0]) * 24 * 60 * 60;
+        }
+        if (typeof segments[1] === "string") {
+            totalSeconds += Number.parseInt(segments[1]) * 60 * 60;
+        }
+        totalSeconds += Number.parseInt(segments[2]) * 60;
+        totalSeconds += Number.parseInt(segments[3]);
+        return totalSeconds;
+    };
+    excludeField = excludeFieldSymbol;
+    get path() {
+        return this.#path;
+    }
+}

package/dist/DomSelection.d.ts

@@ -0,0 +1,30 @@
+import type { Cheerio, CheerioAPI } from "cheerio";
+import type { AnyNode } from "domhandler";
+export declare abstract class DomSelection {
+    abstract select: (selector: string) => DomSelection;
+    abstract attr: (attr: string) => string | undefined;
+    abstract exists: (selector: string) => boolean;
+    abstract get text(): string | Promise<string>;
+    abstract get nativeSelector(): Cheerio<AnyNode>;
+    abstract get selectedNodes(): DomSelection[];
+    abstract get firstJsonLd(): Record<string, unknown>;
+    abstract get jsonLd(): Array<Record<string, unknown>>;
+    map<T>(mapFunction: (node: DomSelection, index: number) => T): T[];
+}
+export declare class CheerioDomSelection extends DomSelection {
+    #private;
+    constructor(cheerioAPI: CheerioAPI, cheerioNode?: Cheerio<AnyNode>);
+    select: (selector: string) => CheerioDomSelection;
+    attr: (attr: string) => string | undefined;
+    exists: (selector: string) => boolean;
+    get nativeSelector(): Cheerio<AnyNode>;
+    get text(): string;
+    map<T>(mapFunction: (node: CheerioDomSelection, index: number) => T): T[];
+    get selectedNodes(): Array<CheerioDomSelection>;
+    get firstJsonLd(): Record<string, unknown>;
+    get canonicalUrl(): string | undefined;
+    get jsonLd(): Array<Record<string, unknown>>;
+    get data(): Record<string, unknown>;
+    get value(): string | undefined | string[];
+}
+//# sourceMappingURL=DomSelection.d.ts.map

package/dist/DomSelection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DomSelection.d.ts","sourceRoot":"","sources":["../src/DomSelection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,8BAAsB,YAAY;IAEhC,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,YAAY,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/C,QAAQ,KAAK,IAAI,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,QAAQ,KAAK,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,QAAQ,KAAK,aAAa,IAAI,YAAY,EAAE,CAAC;IAC7C,QAAQ,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpD,QAAQ,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE;CAGnE;AAED,qBAAa,mBAAoB,SAAQ,YAAY;;gBAKvC,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;IAMlE,MAAM,GAAI,UAAU,MAAM,yBAC6C;IAEvE,IAAI,GAAI,MAAM,MAAM,wBAAqC;IACzD,MAAM,GAAI,UAAU,MAAM,aAC4B;IAEtD,IAAI,cAAc,qBAEjB;IAED,IAAI,IAAI,WAEP;IAED,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE;IAIzE,IAAI,aAAa,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAI9C;IAED,IAAI,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEzC;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,SAAS,CAErC;IAED,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAiB3C;IAED,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAElC;IAED,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAEzC;CACF"}

package/dist/DomSelection.js

@@ -0,0 +1,60 @@
+export class DomSelection {
+    map(mapFunction) {
+        return this.selectedNodes.map(mapFunction);
+    }
+}
+export class CheerioDomSelection extends DomSelection {
+    #nativeSelector;
+    #cachedJsonLdArray;
+    #$;
+    constructor(cheerioAPI, cheerioNode) {
+        super();
+        this.#nativeSelector = cheerioNode ?? cheerioAPI.root();
+        this.#$ = cheerioAPI;
+    }
+    select = (selector) => new CheerioDomSelection(this.#$, this.nativeSelector.find(selector));
+    attr = (attr) => this.#nativeSelector.attr(attr);
+    exists = (selector) => Boolean(this.#nativeSelector.find(selector).length);
+    get nativeSelector() {
+        return this.#nativeSelector;
+    }
+    get text() {
+        return this.#nativeSelector.text();
+    }
+    map(mapFunction) {
+        return this.selectedNodes.map(mapFunction);
+    }
+    get selectedNodes() {
+        return this.#nativeSelector
+            .toArray()
+            .map((node) => new CheerioDomSelection(this.#$, this.#$(node)));
+    }
+    get firstJsonLd() {
+        return this.jsonLd[0];
+    }
+    get canonicalUrl() {
+        return this.select("link[rel=canonical]").attr("href");
+    }
+    get jsonLd() {
+        if (!this.#cachedJsonLdArray) {
+            this.#cachedJsonLdArray = this.select('script[type="application/ld+json"]')
+                .map((ldJsonElm) => ldJsonElm.text)
+                .map((ldJsonText) => {
+                try {
+                    return JSON.parse(ldJsonText);
+                }
+                catch (error) {
+                    console.error("Text is not valid JSON:", ldJsonText);
+                    throw error;
+                }
+            });
+        }
+        return this.#cachedJsonLdArray;
+    }
+    get data() {
+        return this.#nativeSelector.data();
+    }
+    get value() {
+        return this.#nativeSelector.val();
+    }
+}

package/dist/Liase.d.ts

@@ -0,0 +1,40 @@
+import { type FinderOptions, type FinderOptionsInput } from "./schemas/finderOptions.js";
+import type { Plugin } from "./schemas/plugin.js";
+import type { Source } from "./schemas/source.js";
+import type { LiaseHooks } from "./lib/hooks.js";
+import { type RequestHandler } from "./schemas/requestHandler.js";
+export default class Liase {
+    protected sourceMap: {
+        [sourceName: string]: Source;
+    };
+    get sources(): Source[];
+    _finderOptions: FinderOptions;
+    _hooks: LiaseHooks;
+    constructor(finderOptions?: FinderOptionsInput);
+    loadPlugin(plugin: Plugin): void;
+    loadSources(sources: Source[]): void;
+    loadSource(source: Source): void;
+    loadHooks(hooks: NonNullable<Plugin["hooks"]>): void;
+    getSource(sourceId: string): Source;
+    getRequestHandler(sourceId: string, queryType: string): RequestHandler;
+    getRequestSchema(sourceId: string, queryType: string): import("zod").ZodObject<{
+        source: import("zod").ZodString;
+        queryType: import("zod").ZodString;
+        pageNumber?: import("zod").ZodTypeAny;
+        cursor?: import("zod").ZodTypeAny;
+    }, "strict", import("zod").ZodTypeAny, Omit<{
+        source: string;
+        queryType: string;
+        pageNumber?: unknown;
+        cursor?: unknown;
+    }, "pageNumber" | "cursor"> & {
+        pageNumber?: number;
+        cursor?: string | number | null;
+    }, {
+        source: string;
+        queryType: string;
+        pageNumber?: unknown;
+        cursor?: unknown;
+    }>;
+}
+//# sourceMappingURL=Liase.d.ts.map

package/dist/Liase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Liase.d.ts","sourceRoot":"","sources":["../src/Liase.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,kBAAkB,EAExB,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,EACL,KAAK,cAAc,EAEpB,MAAM,6BAA6B,CAAC;AAErC,MAAM,CAAC,OAAO,OAAO,KAAK;IACxB,SAAS,CAAC,SAAS,EAAE;QAAE,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAM;IAC3D,IAAI,OAAO,IAAI,MAAM,EAAE,CAEtB;IAED,cAAc,EAAE,aAAa,CAAC;IAC9B,MAAM,EAAE,UAAU,CAGhB;gBAEU,aAAa,GAAE,kBAAuB;IAQlD,UAAU,CAAC,MAAM,EAAE,MAAM;IASzB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE;IAM7B,UAAU,CAAC,MAAM,EAAE,MAAM;IAmEzB,SAAS,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,IAAI;IAUpD,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAUnC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,cAAc;IAatE,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;CAGrD"}

package/dist/Liase.js

@@ -0,0 +1,105 @@
+import { finderOptionsSchema, } from "./schemas/finderOptions.js";
+import { zodParseOrThrow } from "./lib/zod.js";
+import builtInSourcesPlugin from "./plugins/built-in-sources/index.js";
+import { requestHandlerSchema, } from "./schemas/requestHandler.js";
+export default class Liase {
+    sourceMap = {};
+    get sources() {
+        return Object.values(this.sourceMap);
+    }
+    _finderOptions;
+    _hooks = {
+        loadUrl: [],
+        getFetchClient: [],
+    };
+    constructor(finderOptions = {}) {
+        this._finderOptions = finderOptionsSchema.parse(finderOptions);
+        this.loadPlugin(builtInSourcesPlugin);
+        for (const plugin of this._finderOptions.plugins) {
+            this.loadPlugin(plugin);
+        }
+    }
+    loadPlugin(plugin) {
+        if (plugin.sources) {
+            this.loadSources(plugin.sources);
+        }
+        if (plugin.hooks) {
+            this.loadHooks(plugin.hooks);
+        }
+    }
+    loadSources(sources) {
+        for (const source of sources) {
+            this.loadSource(source);
+        }
+    }
+    loadSource(source) {
+        if (Object.prototype.hasOwnProperty.call(this.sourceMap, source.id)) {
+            console.warn(`Loading "${source.id}" but a source with the same id has already been loaded. The existing source will be overwritten.`);
+        }
+        // Validate request handlers
+        for (const requestHandler of source.requestHandlers) {
+            const { paginationType, requestSchema, responses } = requestHandler;
+            const errorMessage = `Could not load source "${source.id}" as the request handler "${requestHandler.id}" is invalid`;
+            zodParseOrThrow(requestHandlerSchema, requestHandler, { errorMessage });
+            const paginationInvalidRequestError = (issue, property) => new Error(`Request handler "${requestHandler.id}" of source "${source.id}" has paginationType ${paginationType} but ${issue === "missing" ? "is missing" : "includes"} ${property} in the request schema.`);
+            if (paginationType === "offset") {
+                if (!requestSchema.shape.pageNumber) {
+                    throw paginationInvalidRequestError("missing", "pageNumber");
+                }
+                if (requestSchema.shape.cursor) {
+                    throw paginationInvalidRequestError("has", "cursor");
+                }
+            }
+            else if (paginationType === "cursor") {
+                if (!requestSchema.shape.cursor) {
+                    throw paginationInvalidRequestError("missing", "cursor");
+                }
+                if (requestSchema.shape.pageNumber) {
+                    throw paginationInvalidRequestError("has", "pageNumber");
+                }
+            }
+            else if (paginationType === "none") {
+                if (requestSchema.shape.pageNumber) {
+                    throw paginationInvalidRequestError("has", "pageNumber");
+                }
+                if (requestSchema.shape.cursor) {
+                    throw paginationInvalidRequestError("has", "cursor");
+                }
+            }
+            else {
+                throw new Error(`Request handler "${requestHandler.id}" of source "${source.id}" has unsupported paginationType ` +
+                    `${paginationType}`);
+            }
+            const nonDefaultResponseDetails = responses.slice(0, -1);
+            if (nonDefaultResponseDetails.some((responseDetails) => !responseDetails.requestMatcher)) {
+                throw Error(`Some response schema elements are missing "requestMatcher" field for request handler "${requestHandler.id}" of source "${source.id}". ("requestMatcher" is mandatory in all except the last response schema element.`);
+            }
+        }
+        this.sourceMap[source.id] = source;
+    }
+    loadHooks(hooks) {
+        for (const [hookName, hook] of Object.entries(hooks)) {
+            if (hook) {
+                this._hooks[hookName].push(hook);
+            }
+        }
+    }
+    getSource(sourceId) {
+        const source = this.sourceMap[sourceId];
+        if (!source) {
+            throw new Error(`Attempted to query an unknown source. If "${sourceId}" is provided by a plugin please make sure that plugin is loaded first before attempting to query.`);
+        }
+        return source;
+    }
+    getRequestHandler(sourceId, queryType) {
+        const source = this.getSource(sourceId);
+        const handler = source.requestHandlers.find((handler) => handler.id === queryType);
+        if (!handler) {
+            throw new Error(`The source "${sourceId}" does not provide a request handler for the query type "${queryType}".`);
+        }
+        return handler;
+    }
+    getRequestSchema(sourceId, queryType) {
+        return this.getRequestHandler(sourceId, queryType).requestSchema;
+    }
+}