@osqd/bothandlerjs 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +516 -0
- package/LICENSE +102 -0
- package/README.md +308 -0
- package/bin/bothandlerjs.mjs +14 -0
- package/dist/actions/index.d.ts +22 -0
- package/dist/actions/types.d.ts +59 -0
- package/dist/adapters/fastify.d.ts +43 -0
- package/dist/adapters/fetch.d.ts +51 -0
- package/dist/adapters/index.cjs +723 -0
- package/dist/adapters/index.cjs.map +1 -0
- package/dist/adapters/index.d.ts +8 -0
- package/dist/adapters/index.js +692 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/koa.d.ts +27 -0
- package/dist/adapters/node.d.ts +29 -0
- package/dist/adapters/shared.d.ts +39 -0
- package/dist/audit.d.ts +172 -0
- package/dist/challenge/index.d.ts +176 -0
- package/dist/challenge/language.d.ts +57 -0
- package/dist/challenge/page.d.ts +42 -0
- package/dist/challenge/pow.d.ts +33 -0
- package/dist/challenge/token.d.ts +72 -0
- package/dist/cli.cjs +13388 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.js +13358 -0
- package/dist/cli.js.map +1 -0
- package/dist/client/index.cjs +139 -0
- package/dist/client/index.cjs.map +1 -0
- package/dist/client/index.d.ts +63 -0
- package/dist/client/index.js +112 -0
- package/dist/client/index.js.map +1 -0
- package/dist/config.d.ts +316 -0
- package/dist/core.d.ts +443 -0
- package/dist/corpus/adversarial.d.ts +2 -0
- package/dist/corpus/advertising-email.d.ts +2 -0
- package/dist/corpus/ai-crawlers.d.ts +2 -0
- package/dist/corpus/benign-bots.d.ts +2 -0
- package/dist/corpus/cdn-gateways.d.ts +2 -0
- package/dist/corpus/cookies.d.ts +49 -0
- package/dist/corpus/crawlers-regional.d.ts +2 -0
- package/dist/corpus/crawlers-vertical.d.ts +2 -0
- package/dist/corpus/headers.d.ts +403 -0
- package/dist/corpus/humans-apps.d.ts +2 -0
- package/dist/corpus/humans-browsers.d.ts +2 -0
- package/dist/corpus/humans.d.ts +2 -0
- package/dist/corpus/index.cjs +4196 -0
- package/dist/corpus/index.cjs.map +1 -0
- package/dist/corpus/index.d.ts +37 -0
- package/dist/corpus/index.js +4132 -0
- package/dist/corpus/index.js.map +1 -0
- package/dist/corpus/infrastructure.d.ts +17 -0
- package/dist/corpus/libraries-extended.d.ts +2 -0
- package/dist/corpus/ranges.d.ts +31 -0
- package/dist/corpus/reputation.d.ts +21 -0
- package/dist/corpus/runner.d.ts +130 -0
- package/dist/corpus/schema.d.ts +185 -0
- package/dist/corpus/tooling.d.ts +2 -0
- package/dist/corpus/unwanted.d.ts +2 -0
- package/dist/crawler-ranges.d.ts +113 -0
- package/dist/dashboard/client.generated.d.ts +7 -0
- package/dist/dashboard/feed.d.ts +186 -0
- package/dist/dashboard/index.d.ts +5 -0
- package/dist/dashboard/page.d.ts +51 -0
- package/dist/dashboard/parse-request.d.ts +28 -0
- package/dist/dashboard/preview.d.ts +45 -0
- package/dist/dashboard/server.d.ts +70 -0
- package/dist/dashboard/types.d.ts +714 -0
- package/dist/detectors/accept-signature.d.ts +6 -0
- package/dist/detectors/browsing-coherence.d.ts +44 -0
- package/dist/detectors/cadence.d.ts +28 -0
- package/dist/detectors/clearance.d.ts +25 -0
- package/dist/detectors/client-hints.d.ts +19 -0
- package/dist/detectors/client-signals.d.ts +27 -0
- package/dist/detectors/crawl-breadth.d.ts +28 -0
- package/dist/detectors/crawler-verification.d.ts +39 -0
- package/dist/detectors/fetch-metadata.d.ts +20 -0
- package/dist/detectors/header-integrity.d.ts +26 -0
- package/dist/detectors/header-order.d.ts +27 -0
- package/dist/detectors/identity-rotation.d.ts +34 -0
- package/dist/detectors/index.d.ts +56 -0
- package/dist/detectors/ip-intelligence.d.ts +30 -0
- package/dist/detectors/known-bots.d.ts +94 -0
- package/dist/detectors/probe-signature.d.ts +50 -0
- package/dist/detectors/rate-anomaly.d.ts +36 -0
- package/dist/detectors/self-identified.d.ts +33 -0
- package/dist/detectors/session-integrity.d.ts +24 -0
- package/dist/detectors/tls-fingerprint.d.ts +34 -0
- package/dist/detectors/trap.d.ts +98 -0
- package/dist/detectors/types.d.ts +70 -0
- package/dist/detectors/ua-coherence.d.ts +32 -0
- package/dist/evidence.d.ts +45 -0
- package/dist/facts.d.ts +46 -0
- package/dist/index.cjs +9600 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +63 -0
- package/dist/index.js +9444 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/async.d.ts +16 -0
- package/dist/internal/clock.d.ts +17 -0
- package/dist/internal/crypto.d.ts +26 -0
- package/dist/internal/dns.d.ts +100 -0
- package/dist/internal/emitter.d.ts +18 -0
- package/dist/internal/http.d.ts +22 -0
- package/dist/internal/ip.d.ts +73 -0
- package/dist/internal/lru.d.ts +34 -0
- package/dist/internal/matcher.d.ts +41 -0
- package/dist/internal/pattern.d.ts +14 -0
- package/dist/internal/ua.d.ts +92 -0
- package/dist/metrics.d.ts +120 -0
- package/dist/notify/hub.d.ts +70 -0
- package/dist/notify/index.d.ts +7 -0
- package/dist/notify/redact.d.ts +48 -0
- package/dist/notify/sinks.d.ts +62 -0
- package/dist/notify/types.d.ts +56 -0
- package/dist/policy/index.d.ts +7 -0
- package/dist/policy/match.d.ts +20 -0
- package/dist/policy/policy.d.ts +123 -0
- package/dist/policy/presets.d.ts +161 -0
- package/dist/policy/types.d.ts +155 -0
- package/dist/robots.d.ts +83 -0
- package/dist/state.d.ts +161 -0
- package/dist/stores/index.d.ts +5 -0
- package/dist/stores/memory.d.ts +33 -0
- package/dist/stores/redis.d.ts +59 -0
- package/dist/stores/types.d.ts +37 -0
- package/dist/types.d.ts +250 -0
- package/docs/challenge/index.md +152 -0
- package/docs/challenge/localisation.md +105 -0
- package/docs/concepts/actors.md +113 -0
- package/docs/concepts/evidence.md +133 -0
- package/docs/concepts/the-guard.md +121 -0
- package/docs/concepts/threat-model.md +120 -0
- package/docs/concepts/verdicts.md +108 -0
- package/docs/course/01-first-assessment.md +178 -0
- package/docs/course/02-proof-and-suspicion.md +164 -0
- package/docs/course/03-verdicts-and-scores.md +160 -0
- package/docs/course/04-the-guard.md +200 -0
- package/docs/course/05-detectors.md +178 -0
- package/docs/course/06-identity.md +161 -0
- package/docs/course/07-actors.md +178 -0
- package/docs/course/08-traps.md +183 -0
- package/docs/course/09-rules.md +224 -0
- package/docs/course/10-actions-and-presets.md +216 -0
- package/docs/course/11-the-challenge.md +227 -0
- package/docs/course/12-going-live.md +211 -0
- package/docs/course/13-operating-it.md +238 -0
- package/docs/course/14-scaling.md +195 -0
- package/docs/course/15-extending.md +259 -0
- package/docs/course/16-proving-it.md +266 -0
- package/docs/course/index.md +112 -0
- package/docs/design/decisions.md +216 -0
- package/docs/detection/client-signals.md +96 -0
- package/docs/detection/detectors.md +376 -0
- package/docs/detection/index.md +134 -0
- package/docs/detection/signatures.md +111 -0
- package/docs/detection/verification.md +123 -0
- package/docs/detection/writing-a-detector.md +145 -0
- package/docs/index.md +100 -0
- package/docs/integration/adapters.md +138 -0
- package/docs/integration/client-ip.md +114 -0
- package/docs/integration/index.md +55 -0
- package/docs/integration/stores.md +76 -0
- package/docs/operations/audit.md +116 -0
- package/docs/operations/dashboard.md +725 -0
- package/docs/operations/index.md +101 -0
- package/docs/operations/metrics.md +74 -0
- package/docs/operations/notifications.md +72 -0
- package/docs/operations/runtime-changes.md +118 -0
- package/docs/policy/actions.md +196 -0
- package/docs/policy/index.md +84 -0
- package/docs/policy/presets.md +191 -0
- package/docs/policy/robots.md +128 -0
- package/docs/policy/rules.md +134 -0
- package/docs/reference/api.md +126 -0
- package/docs/reference/configuration.md +175 -0
- package/docs/start/choosing-a-policy.md +127 -0
- package/docs/start/first-integration.md +127 -0
- package/docs/start/installation.md +63 -0
- package/docs/start/upgrading.md +72 -0
- package/docs/testing/cli.md +119 -0
- package/docs/testing/corpus.md +129 -0
- package/docs/testing/index.md +61 -0
- package/docs/testing/replay.md +102 -0
- package/docs/testing/try-it.md +189 -0
- package/package.json +145 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Policy } from "../policy/policy.js";
|
|
2
|
+
import type { DashboardEntry, PolicyPreview } from "./types.js";
|
|
3
|
+
import type { Assessment } from "../types.js";
|
|
4
|
+
import type { Rule } from "../policy/types.js";
|
|
5
|
+
import type { GuardSettings } from "../policy/policy.js";
|
|
6
|
+
/**
|
|
7
|
+
* "What would this rule set have done to the traffic I just watched?"
|
|
8
|
+
*
|
|
9
|
+
* This is the `replay` command's question, asked about the last few minutes instead of
|
|
10
|
+
* about a log file, and answered before the change is applied rather than after. The
|
|
11
|
+
* reason it can be answered at all is that {@link Policy.decide} is **pure**: it reads
|
|
12
|
+
* an assessment and returns a decision, touching no state, no store and no network. So
|
|
13
|
+
* a candidate policy can be run over the retained window as many times as somebody
|
|
14
|
+
* cares to edit it, and nothing about the running system moves.
|
|
15
|
+
*
|
|
16
|
+
* **What is faithful, and what is not.** Rule *matching* is exact: every field the
|
|
17
|
+
* matcher reads — verdict, class, identity, category, certainty, score, path, method,
|
|
18
|
+
* detector, prior confirmations — travels on the feed entry, so a rule matches here
|
|
19
|
+
* exactly when it would have matched then. The safety guard is exact too, for the same
|
|
20
|
+
* reason. What a preview cannot tell you is what the *action* would have done: a
|
|
21
|
+
* challenge might have been solved, a rate limit might not have been reached, a custom
|
|
22
|
+
* handler would have run code this does not run. It answers "which rule, and which
|
|
23
|
+
* action", which is the question an edit is about.
|
|
24
|
+
*/
|
|
25
|
+
export declare function previewPolicy(entries: readonly DashboardEntry[], live: Policy, candidate: Policy, warnings?: string[]): PolicyPreview;
|
|
26
|
+
/**
|
|
27
|
+
* Rebuilds enough of an {@link Assessment} for the policy to decide about it.
|
|
28
|
+
*
|
|
29
|
+
* Deliberately built from the feed entry rather than from a retained assessment
|
|
30
|
+
* object. Keeping the real assessments alive would mean holding every request's full
|
|
31
|
+
* header set — cookies included — in memory for as long as the ring is long, to
|
|
32
|
+
* support a feature that only reads a dozen fields. The entry already carries those
|
|
33
|
+
* dozen fields, so the preview costs nothing extra and the headers stay where they
|
|
34
|
+
* were already redacted.
|
|
35
|
+
*/
|
|
36
|
+
export declare function assessmentFromEntry(entry: DashboardEntry): Assessment;
|
|
37
|
+
/**
|
|
38
|
+
* Builds a candidate policy with the live policy's guard settings.
|
|
39
|
+
*
|
|
40
|
+
* The guard is copied rather than accepted from the caller, and that is the point: a
|
|
41
|
+
* preview — and the apply that may follow it — changes which rules exist and never how
|
|
42
|
+
* far one may go. Somebody editing rules in a browser cannot turn `strict` into
|
|
43
|
+
* `aggressive` by adding a field to the JSON, because the field is not read.
|
|
44
|
+
*/
|
|
45
|
+
export declare function candidatePolicy(rules: readonly Rule[], live: Policy, guard?: Partial<GuardSettings>): Policy;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { BotHandler } from "../core.js";
|
|
2
|
+
import type { DashboardHandlerOptions, DashboardOptions, DashboardRequestHandler, DashboardServer } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* The dashboard as a request handler, for mounting on a server you already have.
|
|
5
|
+
*
|
|
6
|
+
* The reason to want this is almost always TLS. `startDashboard` opens a plain HTTP
|
|
7
|
+
* listener, which is right on a laptop and wrong in most production networks: the
|
|
8
|
+
* certificate lives at an ingress, everything has to be reachable under one hostname,
|
|
9
|
+
* or the platform exposes exactly one port. None of that is an argument against the
|
|
10
|
+
* design — it is an argument about *which* server the page is served from.
|
|
11
|
+
*
|
|
12
|
+
* ```ts
|
|
13
|
+
* const dashboard = createDashboardHandler(botHandler, {
|
|
14
|
+
* basePath: "/_bots",
|
|
15
|
+
* auth: { authorize: (req) => sessionFrom(req)?.email ?? false },
|
|
16
|
+
* });
|
|
17
|
+
*
|
|
18
|
+
* https.createServer(tls, (req, res) => {
|
|
19
|
+
* if (req.url?.startsWith("/_bots")) return dashboard(req, res);
|
|
20
|
+
* return app(req, res);
|
|
21
|
+
* }).listen(443);
|
|
22
|
+
* ```
|
|
23
|
+
*
|
|
24
|
+
* Two things are different from the listening form, and both follow from not owning
|
|
25
|
+
* the socket:
|
|
26
|
+
*
|
|
27
|
+
* - **`auth` is required**, including the explicit `auth: false`. The listening form
|
|
28
|
+
* may skip it on `127.0.0.1` because the operating system is then the access
|
|
29
|
+
* control; here there is no bind address to inspect, so nothing can be assumed and
|
|
30
|
+
* the safe assumption is "public".
|
|
31
|
+
* - **`close()` does not close a server it does not own.** It unsubscribes from the
|
|
32
|
+
* engine, ends every event stream and stops the timers. Your server is yours.
|
|
33
|
+
*
|
|
34
|
+
* What has *not* changed is the reason the dashboard is a separate listener from your
|
|
35
|
+
* application: mount it on a server that does not run your bot handler. Serving it
|
|
36
|
+
* from inside the application it reports on means reading the dashboard shows up in
|
|
37
|
+
* the dashboard, and a challenge served to your site can lock you out of the tool you
|
|
38
|
+
* are using to read about it.
|
|
39
|
+
*
|
|
40
|
+
* `basePath` should name the path the page is served under, so the page can build its
|
|
41
|
+
* own URLs. Routing accepts the path with or without that prefix, so it works whether
|
|
42
|
+
* or not your framework strips the mount point before calling this.
|
|
43
|
+
*/
|
|
44
|
+
export declare function createDashboardHandler(handler: BotHandler, options: DashboardHandlerOptions): DashboardRequestHandler;
|
|
45
|
+
/**
|
|
46
|
+
* Starts the dashboard on a listener of its own.
|
|
47
|
+
*
|
|
48
|
+
* Two decisions in here are load-bearing, and both are about the fact that this page
|
|
49
|
+
* is a description of your detection rather than a description of your traffic.
|
|
50
|
+
*
|
|
51
|
+
* **It binds to loopback by default.** The page lists client addresses and, on any
|
|
52
|
+
* row you click, the individual pieces of evidence with the reason each one fired.
|
|
53
|
+
* That is a tuning guide for whoever is scraping you: it names the next signal to fix.
|
|
54
|
+
* A dashboard reachable from the internet is a scraper's fastest route to a
|
|
55
|
+
* header set that passes.
|
|
56
|
+
*
|
|
57
|
+
* **It refuses to bind anywhere else without an explicit decision about access.**
|
|
58
|
+
* Not a warning in a log nobody reads — a `ConfigError` at startup, naming the two
|
|
59
|
+
* ways out: configure `auth`, or write `auth: false` to say you have another control
|
|
60
|
+
* in front of it. The failure mode this prevents is the one where a dashboard goes up
|
|
61
|
+
* on 0.0.0.0 for five minutes during an incident and stays there for two years.
|
|
62
|
+
*
|
|
63
|
+
* It is also, deliberately, a *separate listener* from the site it reports on. Serving
|
|
64
|
+
* it from inside your own application would put it behind the bot handler, where
|
|
65
|
+
* watching the dashboard shows up in the dashboard, and where a challenge served to
|
|
66
|
+
* your site can lock you out of the tool you are using to read about it. To serve it
|
|
67
|
+
* from a listener of your own — behind your own TLS, under your own path — see
|
|
68
|
+
* {@link createDashboardHandler}, which is the same dashboard without the socket.
|
|
69
|
+
*/
|
|
70
|
+
export declare function startDashboard(handler: BotHandler, options?: DashboardOptions): Promise<DashboardServer>;
|