@liase/core 1.0.0

package/dist/test/utils/vitest.js

@@ -0,0 +1,166 @@
+import { createLiaseQuery } from "../../index.js";
+import { getDuplicates, getOrdinal, hasNoDuplicates } from "../../lib/utils.js";
+import { copy } from "copy-anything";
+import deepmerge from "deepmerge";
+import { expect, test } from "vitest";
+import { getSecrets } from "./general.js";
+export function createBasicTestsForRequestHandlers(options) {
+    const { source, queries, queriesShared } = options;
+    for (const requestHandler of source.requestHandlers) {
+        if (!(requestHandler.id in queries)) {
+            throw Error(`No query provided for request handler ${requestHandler.id}`);
+        }
+        const handlerQueries = [queries[requestHandler.id]].flat(1);
+        for (const query of handlerQueries) {
+            const requestPropsWithoutHandlerDetails = {
+                ...query?.request,
+                ...queriesShared?.request,
+            };
+            const request = {
+                source: source.id,
+                queryType: requestHandler.id,
+                ...requestPropsWithoutHandlerDetails,
+            };
+            const timeout = query.timeout ?? queriesShared?.timeout;
+            const formattedQuery = JSON.stringify(requestPropsWithoutHandlerDetails, null, 2).replace(/\n\s*/g, " ");
+            const testName = query.testName ??
+                queriesShared?.testName ??
+                `Run query "${requestHandler.displayName}" with: ${formattedQuery}`;
+            const expectError = query.expectError ?? queriesShared?.expectError;
+            const skip = query.skip ?? queriesShared?.skip;
+            if (skip) {
+                const message = typeof skip === "string"
+                    ? `SKIPPED: ${testName} — ${skip}`
+                    : `SKIPPED: ${testName}`;
+                console.warn(message);
+                test.skip(testName, () => { });
+                continue;
+            }
+            test(testName, async () => {
+                const beforeCallbacks = [query?.before, queriesShared?.before];
+                for (const before of beforeCallbacks) {
+                    await before?.();
+                }
+                const numOfPagesToLoad = query.numOfPagesToLoad ?? 1;
+                const numOfPagesToExpect = query.numOfPagesToExpect ?? numOfPagesToLoad;
+                // biome-ignore lint/suspicious/noExplicitAny: type narrowing helper needs any for constructor check
+                const isPlainObject = (value) => value?.constructor === Object || Array.isArray(value);
+                const deepMergeOptions = {
+                    isMergeableObject: isPlainObject,
+                };
+                const mediaQuery = await createLiaseQuery({
+                    request,
+                    queryOptions: deepmerge.all([
+                        {
+                            cacheNetworkRequests: "always",
+                        },
+                        queriesShared?.queryOptions || {},
+                        query?.queryOptions || {},
+                        {
+                            secrets: {
+                                ...(await getSecrets(request)),
+                            },
+                        },
+                    ], deepMergeOptions),
+                    finderOptions: deepmerge.all([queriesShared?.finderOptions || {}, query?.finderOptions || {}], deepMergeOptions),
+                });
+                const responses = [];
+                let customResponseTestExpectedAssertions = 0;
+                if (expectError) {
+                    expect.assertions(1);
+                    expect(() => mediaQuery.getNext()).rejects.toThrowError(expectError);
+                    return;
+                }
+                for (let i = 0; i < numOfPagesToLoad; i++) {
+                    const response = await mediaQuery.getNext();
+                    responses.push(response);
+                    if (i + 1 < numOfPagesToExpect) {
+                        // If we're expecting more pages then isLastPage should not be "true"
+                        expect(response?.page?.isLastPage, `Expected to receive another response after ${getOrdinal(i + 1)} but page.isLastPage is set to "true"`).not.toBe(true);
+                    }
+                    else if (numOfPagesToExpect < numOfPagesToLoad) {
+                        // If this is the last expected page of content but we've explicitly requested to load more pages after this
+                        // assume that this is the last page and thus isLastPage should not be "false"
+                        expect(response?.page?.isLastPage, `Expected to not receive another response after ${getOrdinal(i + 1)} but page.isLastPage is set to "false"`).not.toBe(false);
+                    }
+                    else {
+                        // It's easy to calculate the number of expected assertions if we include this dummy assertion
+                        expect(true).toBe(true);
+                    }
+                    if (i < numOfPagesToExpect) {
+                        expect(response, `Expected a response for the ${getOrdinal(i + 1)} request but response was null`).not.toBe(null);
+                    }
+                    else {
+                        expect(response, `Expected null as the response for the ${getOrdinal(i + 1)} request`).toBe(null);
+                    }
+                    const customResponseChecks = [
+                        ...(query?.checkResponse ? [query?.checkResponse] : []),
+                        ...(queriesShared?.checkResponse
+                            ? [queriesShared?.checkResponse]
+                            : []),
+                    ];
+                    for (const checkResponse of customResponseChecks) {
+                        const result = checkResponse(response, {
+                            message: `The response for the ${getOrdinal(i + 1)} request was not what was expected`,
+                            pageLoadNum: i,
+                        });
+                        customResponseTestExpectedAssertions +=
+                            typeof result === "number" ? result : 1;
+                    }
+                    if (!query.duplicateMediaPossible) {
+                        const idsOfMedia = (response?.media || [])
+                            .filter((media) => media)
+                            .map((media) => media.id);
+                        expect(idsOfMedia).toSatisfy(hasNoDuplicates, `Media with the same ID appears in single response: ${getDuplicates(idsOfMedia).join(",")}`);
+                    }
+                }
+                const customAllResponsesChecks = [
+                    ...(query?.checkAllResponses ? [query?.checkAllResponses] : []),
+                    ...(queriesShared?.checkAllResponses
+                        ? [queriesShared?.checkAllResponses]
+                        : []),
+                ];
+                for (const checkAllResponses of customAllResponsesChecks) {
+                    const result = checkAllResponses(responses, {
+                        message: "The responses were not as expected",
+                    });
+                    customResponseTestExpectedAssertions +=
+                        typeof result === "number" ? result : 1;
+                }
+                if (!query.duplicateMediaPossible) {
+                    const idsOfMedia = responses
+                        .flatMap((response) => response?.media || [])
+                        .filter((media) => media)
+                        .map((media) => media.id);
+                    expect(idsOfMedia).toSatisfy(hasNoDuplicates, `Media with the same ID appears in multiple responses: ${getDuplicates(idsOfMedia).join(",")}`);
+                }
+                expect.assertions(numOfPagesToLoad * 2 +
+                    (query.duplicateMediaPossible ? 0 : numOfPagesToLoad + 1) +
+                    customResponseTestExpectedAssertions);
+            }, timeout);
+        }
+    }
+}
+// Some values in a response may naturally change over time
+// or be differ based on other factors like a client's ip.
+// This can result in the tests failing due to a response
+// not matching its snapshot even though this difference may
+// not indicate any problem. To avoid this we first normalise
+// any parts of a response which may naturally vary before we
+// snapshot it.
+export function normaliseResponse(response) {
+    const clonedResponse = copy(response);
+    for (const media of clonedResponse.media || []) {
+        if (media.url) {
+            const url = new URL(media.url);
+            media.url = url.origin + url.pathname;
+        }
+        for (const file of media?.files || []) {
+            if (file.url) {
+                const url = new URL(file.url);
+                file.url = url.origin + url.pathname;
+            }
+        }
+    }
+    return clonedResponse;
+}

package/dist/types.d.ts

@@ -0,0 +1,16 @@
+import type { LiaseHooks } from "./lib/hooks.js";
+import type { QueryOptions } from "./schemas/queryOptions.js";
+import type { GenericRequest } from "./schemas/request.js";
+import type { RequestHandler } from "./schemas/requestHandler.js";
+import type { GenericSecrets } from "./schemas/secrets.js";
+export type ConstructorExecutionContext = {
+    request: GenericRequest;
+    secrets: GenericSecrets;
+    requestHandler: RequestHandler;
+    responseDetails: RequestHandler["responses"][0];
+    pageFetchLimitReached?: boolean;
+    cacheNetworkRequests?: QueryOptions["cacheNetworkRequests"];
+    sourceId: string;
+    hooks: LiaseHooks;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,MAAM,MAAM,2BAA2B,GAAG;IACxC,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,cAAc,CAAC;IACxB,cAAc,EAAE,cAAc,CAAC;IAC/B,eAAe,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,oBAAoB,CAAC,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,UAAU,CAAC;CACnB,CAAC"}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@liase/core",
+  "version": "1.0.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "The easiest way to programmatically fetch media content details on the internet",
+  "main": "dist/index.js",
+  "files": [
+    "dist/*"
+  ],
+  "scripts": {
+    "build": "rm -rf ./dist && tsc -p ./tsconfig.dist.json && tsc -p test/tsconfig.dist.json && tsc-alias",
+    "watch": "tsc -p ./tsconfig.dist.json --watch",
+    "lint": "biome check",
+    "lint:fix": "biome check --fix",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "preview": "test/preview/start.sh"
+  },
+  "keywords": [
+    "media"
+  ],
+  "author": "Callum Gare <callum@gare.au>",
+  "license": "MIT",
+  "type": "module",
+  "dependencies": {
+    "@atproto/api": "^0.13.19",
+    "@refactorjs/http-proxy": "^0.2.14",
+    "cacache": "^19.0.1",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.0.0",
+    "commander": "^12.0.0",
+    "deepmerge": "^4.3.1",
+    "dotenv-flow": "^4.1.0",
+    "entities": "^4.5.0",
+    "got-scraping": "^4.0.8",
+    "json-stable-stringify": "^1.2.1",
+    "mime-types": "^2.1.28",
+    "open": "^10.1.0",
+    "pluralize": "^8.0.0",
+    "tsx": "^4.15.9",
+    "type-fest": "^4.23.0",
+    "zod": "^3.21.4"
+  },
+  "devDependencies": {
+    "@loopback/http-caching-proxy": "^6.0.3",
+    "@tsconfig/node21": "^21.0.3",
+    "@types/cacache": "^17.0.2",
+    "@types/make-fetch-happen": "^10.0.4",
+    "@types/mime-types": "^2.1.1",
+    "@types/node": "^22.1.0",
+    "@types/pluralize": "^0.0.33",
+    "copy-anything": "^4.0.2",
+    "semantic-release": "^24.0.0",
+    "semantic-release-monorepo": "^8.0.2",
+    "tsc-alias": "^1.8.8",
+    "typescript": "^5.5.3",
+    "vitest": "^1.6.0"
+  },
+  "optionalDependencies": {
+    "fsevents": "^2.3.2"
+  },
+  "overrides": {
+    "conventional-changelog-conventionalcommits": ">= 8.0.0"
+  }
+}