@arcjet/analyze 1.0.0-alpha.9 → 1.0.0-beta.10

package/README.md

@@ -1,7 +1,7 @@
 <a href="https://arcjet.com" target="_arcjet-home">
   <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://arcjet.com/arcjet-logo-dark-planet-arrival.svg">
-    <img src="https://arcjet.com/arcjet-logo-light-planet-arrival.svg" alt="Arcjet Logo" height="144" width="auto">
+    <source media="(prefers-color-scheme: dark)" srcset="https://arcjet.com/logo/arcjet-dark-lockup-voyage-horizontal.svg">
+    <img src="https://arcjet.com/logo/arcjet-light-lockup-voyage-horizontal.svg" alt="Arcjet Logo" height="128" width="auto">
   </picture>
 </a>
 
@@ -17,62 +17,76 @@
 </p>
 
 [Arcjet][arcjet] helps developers protect their apps in just a few lines of
-code. Implement rate limiting, bot protection, email verification & defend
+code. Implement rate limiting, bot protection, email verification, and defense
 against common attacks.
 
 This is the [Arcjet][arcjet] local analysis engine.
 
-## Installation
+- [npm package (`@arcjet/analyze`)](https://www.npmjs.com/package/@arcjet/analyze)
+- [GitHub source code (`analyze/` in `arcjet/arcjet-js`)](https://github.com/arcjet/arcjet-js/tree/main/analyze)
 
-```shell
-npm install -S @arcjet/analyze
-```
-
-## Example
-
-```ts
-import { generateFingerprint, isValidEmail } from "@arcjet/analyze";
+## What is this?
 
-const fingerprint = generateFingerprint("127.0.0.1");
-console.log("fingerprint: ", fingerprint);
+This package provides functionality to analyze requests.
+The work is done in WebAssembly but is called here from JavaScript.
+The functionality is wrapped up into rules in our core package
+([`arcjet`][github-arcjet-arcjet]),
+in turn exposed from our adapters (such as `@arcjet/next`).
 
-const valid = isValidEmail("hello@example.com");
-console.log("is email valid?", valid);
-```
+<!-- TODO(@wooorm-arcjet): link `adapters` above when the main repo is up to date. -->
 
-## Implementation
+The WebAssembly files are in
+[`@arcjet/analyze-wasm`][github-arcjet-analyze-wasm].
+They are separate because we need to change the import structure for each
+runtime that we support in the bindings.
+Separate packages lets us not duplicate code while providing a combined
+higher-level API for calling our core functionality.
 
-This package provides analyze logic implemented as a WebAssembly module which
-will run local analysis on request details before calling the Arcjet API.
+## When should I use this?
 
-The [arcjet.wasm.js](./wasm/arcjet.wasm.js) file contains the binary inlined as
-a base64 [Data URL][mdn-data-url] with the `application/wasm` MIME type.
+This is an internal Arcjet package not designed for public use.
+See our [_Get started_ guide][arcjet-get-started] for how to use Arcjet in your
+application.
 
-This was chosen to save on storage space over inlining the file directly as a
-Uint8Array, which would take up ~3x the space of the Wasm file. See
-[Better Binary Batter: Mixing Base64 and Uint8Array][wasm-base64-blog] for more
-details.
+## Install
 
-It is then decoded into an ArrayBuffer to be used directly via WebAssembly's
-`compile()` function in our entry point file.
+This package is ESM only.
+Install with npm in Node.js:
 
-This is all done to avoid trying to read or bundle the Wasm asset in various
-ways based on the platform or bundler a user is targeting. One example being
-that Next.js requires special `asyncWebAssembly` webpack config to load our
-Wasm file if we don't do this.
+```sh
+npm install @arcjet/analyze
+```
 
-In the future, we hope to do away with this workaround when all bundlers
-properly support consistent asset bundling techniques.
+## Use
 
-## API
+```js
+import { generateFingerprint, isValidEmail } from "@arcjet/analyze";
 
-In progress.
+const fingerprint = await generateFingerprint(
+  { characteristics: [] },
+  { ip: "127.0.0.1" },
+);
+console.log(fingerprint);
+// => "fp::2::0d219da6100b99f95cf639b77e088c6df3c096aa5fd61dec5287c5cf94d5e545"
+
+const result = await isValidEmail({}, "hello@example.com", {
+  tag: "allow-email-validation-config",
+  val: {
+    allowDomainLiteral: false,
+    allow: [],
+    requireTopLevelDomain: true,
+  },
+});
+console.log(result);
+// => { blocked: [], validity: "valid" }
+```
 
 ## License
 
-Licensed under the [Apache License, Version 2.0][apache-license].
+[Apache License, Version 2.0][apache-license] © [Arcjet Labs, Inc.][arcjet]
 
 [arcjet]: https://arcjet.com
-[mdn-data-url]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs
-[wasm-base64-blog]: https://blobfolio.com/2019/better-binary-batter-mixing-base64-and-uint8array/
+[arcjet-get-started]: https://docs.arcjet.com/get-started
 [apache-license]: http://www.apache.org/licenses/LICENSE-2.0
+[github-arcjet-analyze-wasm]: https://github.com/arcjet/arcjet-js/tree/main/analyze-wasm
+[github-arcjet-arcjet]: https://github.com/arcjet/arcjet-js/tree/main/arcjet

package/index.d.ts

@@ -1,27 +1,29 @@
-import { type EmailValidationConfig } from "./wasm/arcjet_analyze_js_req.js";
-export { type EmailValidationConfig };
+import type { BotConfig, BotResult, DetectedSensitiveInfoEntity, DetectSensitiveInfoFunction, EmailValidationConfig, EmailValidationResult, SensitiveInfoEntities, SensitiveInfoEntity, SensitiveInfoResult } from "@arcjet/analyze-wasm";
+import type { ArcjetLogger } from "@arcjet/protocol";
+interface AnalyzeContext {
+    log: ArcjetLogger;
+    characteristics: string[];
+}
+type AnalyzeRequest = {
+    ip?: string;
+    method?: string;
+    protocol?: string;
+    host?: string;
+    path?: string;
+    headers?: Record<string, string>;
+    cookies?: string;
+    query?: string;
+    extra?: Record<string, string>;
+};
+export { type EmailValidationConfig, type BotConfig, type SensitiveInfoEntity, type DetectedSensitiveInfoEntity, };
 /**
  * Generate a fingerprint for the client. This is used to identify the client
  * across multiple requests.
- * @param ip - The IP address of the client.
+ * @param context - The Arcjet Analyze context.
+ * @param request - The request to fingerprint.
  * @returns A SHA-256 string fingerprint.
  */
-export declare function generateFingerprint(ip: string): Promise<string>;
-export declare function isValidEmail(candidate: string, options?: EmailValidationConfig): Promise<boolean>;
-/**
- * Represents the result of the bot detection.
- *
- * @property `bot_type` - What type of bot this is. This will be one of `BotType`.
- * @property `bot_score` - A score ranging from 0 to 99 representing the degree of
- * certainty. The higher the number within the type category, the greater the
- * degree of certainty. E.g. `BotType.Automated` with a score of 1 means we are
- * sure the request was made by an automated bot. `BotType.LikelyNotABot` with a
- * score of 30 means we don't think this request was a bot, but it's lowest
- * confidence level. `BotType.LikelyNotABot` with a score of 99 means we are
- * almost certain this request was not a bot.
- */
-export interface BotResult {
-    bot_type: number;
-    bot_score: number;
-}
-export declare function detectBot(headers: string, patterns_add: string, patterns_remove: string): Promise<BotResult>;
+export declare function generateFingerprint(context: AnalyzeContext, request: AnalyzeRequest): Promise<string>;
+export declare function isValidEmail(context: AnalyzeContext, candidate: string, options: EmailValidationConfig): Promise<EmailValidationResult>;
+export declare function detectBot(context: AnalyzeContext, request: AnalyzeRequest, options: BotConfig): Promise<BotResult>;
+export declare function detectSensitiveInfo(context: AnalyzeContext, candidate: string, entities: SensitiveInfoEntities, contextWindowSize: number, detect?: DetectSensitiveInfoFunction): Promise<SensitiveInfoResult>;

package/index.js

@@ -1,109 +1,118 @@
-import initWasm, { detect_bot, generate_fingerprint, is_valid_email } from './wasm/arcjet_analyze_js_req.js';
+import { initializeWasm } from '@arcjet/analyze-wasm';
 
-let state = "uninitialized";
-/**
- * Initialize the WASM module. This can be explicitly called after creating
- * the client, but it will be called automatically if it has not been called
- * when the first request is made. This uses a factory-style pattern because
- * the call must be async and the constructor cannot be async.
- */
-async function init() {
-    if (state === "errored" || state === "unsupported")
-        return;
-    if (typeof WebAssembly === "undefined") {
-        state = "unsupported";
-        return;
-    }
-    if (state === "uninitialized") {
-        try {
-            let wasmModule;
-            // We use `NEXT_RUNTIME` env var to DCE the Node/Browser code in the `else` block
-            // possible values: "edge" | "nodejs" | undefined
-            if (process.env["NEXT_RUNTIME"] === "edge") {
-                const mod = await import(
-                // @ts-expect-error
-                './wasm/arcjet_analyze_js_req_bg.wasm?module');
-                wasmModule = mod.default;
-            }
-            else {
-                const { wasm } = await import('./wasm/arcjet.wasm.js');
-                wasmModule = await WebAssembly.compile(await wasm());
-            }
-            await initWasm(wasmModule);
-            state = "initialized";
-        }
-        catch (err) {
-            state = "errored";
-            return;
-        }
+const FREE_EMAIL_PROVIDERS = [
+    "gmail.com",
+    "yahoo.com",
+    "hotmail.com",
+    "aol.com",
+    "hotmail.co.uk",
+];
+function noOpSensitiveInfoDetect() {
+    return [];
+}
+function noOpBotsDetect() {
+    return [];
+}
+function createCoreImports(detect) {
+    if (typeof detect !== "function") {
+        detect = noOpSensitiveInfoDetect;
     }
     return {
-        detectBot: detect_bot,
-        fingerprint: generate_fingerprint,
-        isValidEmail: is_valid_email,
+        "arcjet:js-req/bot-identifier": {
+            detect: noOpBotsDetect,
+        },
+        "arcjet:js-req/email-validator-overrides": {
+            isFreeEmail(domain) {
+                if (FREE_EMAIL_PROVIDERS.includes(domain)) {
+                    return "yes";
+                }
+                return "unknown";
+            },
+            isDisposableEmail() {
+                return "unknown";
+            },
+            hasMxRecords() {
+                return "unknown";
+            },
+            hasGravatar() {
+                return "unknown";
+            },
+        },
+        // TODO(@wooorm-arcjet): figure out a test case for this with the default `detect`.
+        "arcjet:js-req/sensitive-information-identifier": {
+            detect,
+        },
+        // TODO(@wooorm-arcjet): figure out a test case for this that calls `verify`.
+        "arcjet:js-req/verify-bot": {
+            verify() {
+                return "unverifiable";
+            },
+        },
     };
 }
+// TODO(@wooorm-arcjet): document what is used to fingerprint.
 /**
  * Generate a fingerprint for the client. This is used to identify the client
  * across multiple requests.
- * @param ip - The IP address of the client.
+ * @param context - The Arcjet Analyze context.
+ * @param request - The request to fingerprint.
  * @returns A SHA-256 string fingerprint.
  */
-async function generateFingerprint(ip) {
-    if (ip == "") {
-        return "";
-    }
-    // We use `NEXT_RUNTIME` env var to DCE the JS fallback code in the `else` block
-    // possible values: "edge" | "nodejs" | undefined
-    if (process.env["NEXT_RUNTIME"] === "edge") {
-        const analyze = await init();
-        // We HAVE to have the WasmAPI in Edge
-        const fingerprint = analyze.fingerprint(ip);
-        return fingerprint;
-    }
-    else {
-        const analyze = await init();
-        if (typeof analyze !== "undefined") {
-            const fingerprint = analyze.fingerprint(ip);
-            return fingerprint;
-        }
-        else {
-            // Conditional import because it's not available in some runtimes, we know
-            // it is when running on Vercel serverless functions.
-            // TODO(#180): Avoid nodejs-specific import
-            const createHash = await import('crypto');
-            // Fingerprint v1 is just the IP address
-            const fingerprintRaw = `fp_1_${ip}`;
-            const fingerprint = createHash
-                .createHash("sha256")
-                .update(fingerprintRaw)
-                .digest("hex");
-            return fingerprint;
-        }
+async function generateFingerprint(context, request) {
+    const { log } = context;
+    const coreImports = createCoreImports();
+    const analyze = await initializeWasm(coreImports);
+    if (typeof analyze !== "undefined") {
+        return analyze.generateFingerprint(JSON.stringify(request), context.characteristics);
+        // Ignore the `else` branch as we test in places that have WebAssembly.
+        /* node:coverage ignore next 4 */
     }
+    log.debug("WebAssembly is not supported in this runtime");
+    return "";
 }
-async function isValidEmail(candidate, options) {
-    const analyze = await init();
+// TODO(@wooorm-arcjet): docs.
+async function isValidEmail(context, candidate, options) {
+    const { log } = context;
+    const coreImports = createCoreImports();
+    const analyze = await initializeWasm(coreImports);
     if (typeof analyze !== "undefined") {
         return analyze.isValidEmail(candidate, options);
+        // Ignore the `else` branch as we test in places that have WebAssembly.
+        /* node:coverage ignore next 4 */
     }
-    else {
-        // TODO: Fallback to JS if we don't have WASM?
-        return true;
-    }
+    log.debug("WebAssembly is not supported in this runtime");
+    return { blocked: [], validity: "valid" };
 }
-async function detectBot(headers, patterns_add, patterns_remove) {
-    const analyze = await init();
+// TODO(@wooorm-arcjet): docs.
+async function detectBot(context, request, options) {
+    const { log } = context;
+    const coreImports = createCoreImports();
+    const analyze = await initializeWasm(coreImports);
     if (typeof analyze !== "undefined") {
-        return analyze.detectBot(headers, patterns_add, patterns_remove);
+        return analyze.detectBot(JSON.stringify(request), options);
+        // Ignore the `else` branch as we test in places that have WebAssembly.
+        /* node:coverage ignore next 4 */
     }
-    else {
-        // TODO: Fallback to JS if we don't have WASM?
-        return {
-            bot_type: 1, // NOT_ANALYZED
-            bot_score: 0,
-        };
+    log.debug("WebAssembly is not supported in this runtime");
+    return { allowed: [], denied: [], spoofed: false, verified: false };
+}
+// TODO(@wooorm-arcjet): docs.
+async function detectSensitiveInfo(context, candidate, entities, contextWindowSize, detect) {
+    const { log } = context;
+    const coreImports = createCoreImports(detect);
+    const analyze = await initializeWasm(coreImports);
+    if (typeof analyze !== "undefined") {
+        const skipCustomDetect = typeof detect !== "function";
+        return analyze.detectSensitiveInfo(candidate, {
+            entities,
+            contextWindowSize,
+            skipCustomDetect,
+        });
+        // Ignore the `else` branch as we test in places that have WebAssembly.
+        /* node:coverage ignore next 4 */
     }
+    log.debug("WebAssembly is not supported in this runtime");
+    throw new Error("SENSITIVE_INFO rule failed to run because Wasm is not supported in this environment.");
 }
 
-export { detectBot, generateFingerprint, isValidEmail };
+export { detectBot, detectSensitiveInfo, generateFingerprint, isValidEmail };

package/package.json

@@ -1,7 +1,15 @@
 {
   "name": "@arcjet/analyze",
-  "version": "1.0.0-alpha.9",
+  "version": "1.0.0-beta.10",
   "description": "Arcjet local analysis engine",
+  "keywords": [
+    "analyze",
+    "arcjet",
+    "attack",
+    "limit",
+    "protect",
+    "verify"
+  ],
   "license": "Apache-2.0",
   "homepage": "https://arcjet.com",
   "repository": {
@@ -25,36 +33,30 @@
   "main": "./index.js",
   "types": "./index.d.ts",
   "files": [
-    "LICENSE",
-    "README.md",
-    "wasm/",
-    "*.js",
-    "*.d.ts",
-    "*.ts",
-    "!*.config.js"
+    "index.d.ts",
+    "index.js"
   ],
   "scripts": {
-    "prepublishOnly": "npm run build",
     "build": "rollup --config rollup.config.js",
     "lint": "eslint .",
-    "pretest": "npm run build",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --passWithNoTests"
+    "prepublishOnly": "npm run build",
+    "test-api": "node --test",
+    "test-coverage": "node --experimental-test-coverage --test",
+    "test": "npm run build && npm run lint && npm run test-coverage"
   },
-  "sideEffects": [
-    "./wasm/arcjet_analyze_js_req_bg.js"
-  ],
   "dependencies": {
-    "@arcjet/logger": "1.0.0-alpha.9"
+    "@arcjet/analyze-wasm": "1.0.0-beta.10",
+    "@arcjet/protocol": "1.0.0-beta.10"
   },
   "devDependencies": {
-    "@arcjet/eslint-config": "1.0.0-alpha.9",
-    "@arcjet/rollup-config": "1.0.0-alpha.9",
-    "@arcjet/tsconfig": "1.0.0-alpha.9",
-    "@jest/globals": "29.7.0",
-    "@rollup/wasm-node": "4.12.0",
+    "@arcjet/eslint-config": "1.0.0-beta.10",
+    "@arcjet/rollup-config": "1.0.0-beta.10",
+    "@arcjet/tsconfig": "1.0.0-beta.10",
+    "@bytecodealliance/jco": "1.5.0",
+    "@rollup/wasm-node": "4.46.2",
     "@types/node": "18.18.0",
-    "jest": "29.7.0",
-    "typescript": "5.3.3"
+    "eslint": "9.32.0",
+    "typescript": "5.9.2"
   },
   "publishConfig": {
     "access": "public",

package/index.ts

@@ -1,165 +0,0 @@
-import initWasm, {
-  detect_bot,
-  generate_fingerprint,
-  is_valid_email,
-  type EmailValidationConfig,
-} from "./wasm/arcjet_analyze_js_req.js";
-
-export { type EmailValidationConfig };
-
-type WasmAPI = {
-  /**
-   * The WASM detect_bot function. Initialized by calling `init()`. Defined at a
-   * class level to avoid having to load the WASM module multiple times.
-   */
-  detectBot: typeof detect_bot;
-  /**
-   * The WASM fingerprint function. Initialized by calling `init()`. Defined at
-   * a class level to avoid having to load the WASM module multiple times.
-   */
-  fingerprint: typeof generate_fingerprint;
-  /**
-   * The WASM email validation function. Initialized by calling `init()`. Defined at
-   * a class level to avoid having to load the WASM module multiple times.
-   */
-  isValidEmail: typeof is_valid_email;
-};
-
-type WasmState = "initialized" | "uninitialized" | "unsupported" | "errored";
-
-let state: WasmState = "uninitialized";
-
-/**
- * Initialize the WASM module. This can be explicitly called after creating
- * the client, but it will be called automatically if it has not been called
- * when the first request is made. This uses a factory-style pattern because
- * the call must be async and the constructor cannot be async.
- */
-async function init(): Promise<WasmAPI | undefined> {
-  if (state === "errored" || state === "unsupported") return;
-
-  if (typeof WebAssembly === "undefined") {
-    state = "unsupported";
-    return;
-  }
-
-  if (state === "uninitialized") {
-    try {
-      let wasmModule: WebAssembly.Module;
-      // We use `NEXT_RUNTIME` env var to DCE the Node/Browser code in the `else` block
-      // possible values: "edge" | "nodejs" | undefined
-      if (process.env["NEXT_RUNTIME"] === "edge") {
-        const mod = await import(
-          // @ts-expect-error
-          "./wasm/arcjet_analyze_js_req_bg.wasm?module"
-        );
-        wasmModule = mod.default;
-      } else {
-        const { wasm } = await import("./wasm/arcjet.wasm.js");
-        wasmModule = await WebAssembly.compile(await wasm());
-      }
-
-      await initWasm(wasmModule);
-      state = "initialized";
-    } catch (err) {
-      state = "errored";
-      return;
-    }
-  }
-
-  return {
-    detectBot: detect_bot,
-    fingerprint: generate_fingerprint,
-    isValidEmail: is_valid_email,
-  };
-}
-
-/**
- * Generate a fingerprint for the client. This is used to identify the client
- * across multiple requests.
- * @param ip - The IP address of the client.
- * @returns A SHA-256 string fingerprint.
- */
-export async function generateFingerprint(ip: string): Promise<string> {
-  if (ip == "") {
-    return "";
-  }
-
-  // We use `NEXT_RUNTIME` env var to DCE the JS fallback code in the `else` block
-  // possible values: "edge" | "nodejs" | undefined
-  if (process.env["NEXT_RUNTIME"] === "edge") {
-    const analyze = await init();
-    // We HAVE to have the WasmAPI in Edge
-    const fingerprint = analyze!.fingerprint(ip);
-    return fingerprint;
-  } else {
-    const analyze = await init();
-    if (typeof analyze !== "undefined") {
-      const fingerprint = analyze.fingerprint(ip);
-      return fingerprint;
-    } else {
-      // Conditional import because it's not available in some runtimes, we know
-      // it is when running on Vercel serverless functions.
-      // TODO(#180): Avoid nodejs-specific import
-      const createHash = await import("crypto");
-
-      // Fingerprint v1 is just the IP address
-      const fingerprintRaw = `fp_1_${ip}`;
-
-      const fingerprint = createHash
-        .createHash("sha256")
-        .update(fingerprintRaw)
-        .digest("hex");
-      return fingerprint;
-    }
-  }
-}
-
-export async function isValidEmail(
-  candidate: string,
-  options?: EmailValidationConfig,
-) {
-  const analyze = await init();
-
-  if (typeof analyze !== "undefined") {
-    return analyze.isValidEmail(candidate, options);
-  } else {
-    // TODO: Fallback to JS if we don't have WASM?
-    return true;
-  }
-}
-
-/**
- * Represents the result of the bot detection.
- *
- * @property `bot_type` - What type of bot this is. This will be one of `BotType`.
- * @property `bot_score` - A score ranging from 0 to 99 representing the degree of
- * certainty. The higher the number within the type category, the greater the
- * degree of certainty. E.g. `BotType.Automated` with a score of 1 means we are
- * sure the request was made by an automated bot. `BotType.LikelyNotABot` with a
- * score of 30 means we don't think this request was a bot, but it's lowest
- * confidence level. `BotType.LikelyNotABot` with a score of 99 means we are
- * almost certain this request was not a bot.
- */
-export interface BotResult {
-  bot_type: number;
-  bot_score: number;
-}
-
-export async function detectBot(
-  headers: string,
-  patterns_add: string,
-  patterns_remove: string,
-): Promise<BotResult> {
-  const analyze = await init();
-
-  if (typeof analyze !== "undefined") {
-    return analyze.detectBot(headers, patterns_add, patterns_remove);
-  } else {
-    // TODO: Fallback to JS if we don't have WASM?
-    return {
-      bot_type: 1, // NOT_ANALYZED
-      bot_score: 0,
-    };
-  }
-}

package/wasm/arcjet.wasm.d.ts

@@ -1,31 +0,0 @@
-// @generated by wasm2module - DO NOT EDIT
-/* tslint:disable */
-/* eslint-disable */
-
-/**
- * This file contains the Arcjet Wasm binary inlined as a base64
- * [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs)
- * with the application/wasm MIME type.
- *
- * This was chosen to save on storage space over inlining the file directly as
- * a Uint8Array, which would take up ~3x the space of the Wasm file. See
- * https://blobfolio.com/2019/better-binary-batter-mixing-base64-and-uint8array/
- * for more details.
- *
- * It is then decoded into an ArrayBuffer to be used directly via WebAssembly's
- * `compile()` function in our entry point file.
- *
- * This is all done to avoid trying to read or bundle the Wasm asset in various
- * ways based on the platform or bundler a user is targeting. One example being
- * that Next.js requires special `asyncWebAssembly` webpack config to load our
- * Wasm file if we don't do this.
- *
- * In the future, we hope to do away with this workaround when all bundlers
- * properly support consistent asset bundling techniques.
- */
-
-/**
- * Returns an ArrayBuffer for the Arcjet Wasm binary, decoded from a base64 Data
- * URL.
- */
-export function wasm(): Promise<ArrayBuffer>;