@sdxc/spam 0.0.0-pre.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE.md +21 -0
  2. package/README.md +382 -0
  3. package/dist/akismet.d.ts +63 -0
  4. package/dist/akismet.js +127 -0
  5. package/dist/author-email.d.ts +16 -0
  6. package/dist/author-email.js +42 -0
  7. package/dist/bayes/classify.d.ts +39 -0
  8. package/dist/bayes/classify.js +72 -0
  9. package/dist/bayes/data-table.d.ts +58 -0
  10. package/dist/bayes/data-table.js +123 -0
  11. package/dist/bayes/memory.d.ts +18 -0
  12. package/dist/bayes/memory.js +26 -0
  13. package/dist/bayes/store.d.ts +41 -0
  14. package/dist/bayes/store.js +0 -0
  15. package/dist/bayes/tokens.d.ts +31 -0
  16. package/dist/bayes/tokens.js +42 -0
  17. package/dist/bayes.d.ts +43 -0
  18. package/dist/bayes.js +67 -0
  19. package/dist/check.d.ts +116 -0
  20. package/dist/check.js +20 -0
  21. package/dist/corpus.fixtures.d.ts +20 -0
  22. package/dist/corpus.fixtures.js +233 -0
  23. package/dist/filter.d.ts +70 -0
  24. package/dist/filter.js +122 -0
  25. package/dist/index.d.ts +13 -0
  26. package/dist/index.js +11 -0
  27. package/dist/lib/text.d.ts +16 -0
  28. package/dist/lib/text.js +43 -0
  29. package/dist/link-blocklist.d.ts +64 -0
  30. package/dist/link-blocklist.js +202 -0
  31. package/dist/memory.d.ts +71 -0
  32. package/dist/memory.js +73 -0
  33. package/dist/rules/author-name.d.ts +29 -0
  34. package/dist/rules/author-name.js +39 -0
  35. package/dist/rules/contact-bait.d.ts +28 -0
  36. package/dist/rules/contact-bait.js +43 -0
  37. package/dist/rules/index.d.ts +24 -0
  38. package/dist/rules/index.js +26 -0
  39. package/dist/rules/language.d.ts +29 -0
  40. package/dist/rules/language.js +84 -0
  41. package/dist/rules/link-syntax.d.ts +28 -0
  42. package/dist/rules/link-syntax.js +48 -0
  43. package/dist/rules/link-targets.d.ts +41 -0
  44. package/dist/rules/link-targets.js +87 -0
  45. package/dist/rules/links.d.ts +32 -0
  46. package/dist/rules/links.js +39 -0
  47. package/dist/rules/shouting.d.ts +31 -0
  48. package/dist/rules/shouting.js +60 -0
  49. package/dist/rules/timing.d.ts +29 -0
  50. package/dist/rules/timing.js +46 -0
  51. package/dist/rules/unicode.d.ts +31 -0
  52. package/dist/rules/unicode.js +66 -0
  53. package/dist/stop-forum-spam.d.ts +52 -0
  54. package/dist/stop-forum-spam.js +135 -0
  55. package/dist/workers-ai.d.ts +68 -0
  56. package/dist/workers-ai.js +164 -0
  57. package/package.json +32 -0
@@ -0,0 +1,39 @@
1
+ /**
2
+ * The arithmetic of the classifier: Robinson's smoothed per-token spam probability and Fisher's
3
+ * method for combining the most telling tokens into one probability. Kept free of storage so the
4
+ * numbers can be checked on their own.
5
+ *
6
+ * @author [Sergio Xalambrí](https://sergiodxa.com)
7
+ * @copyright Sergio Xalambrí 2026
8
+ */
9
+ import type { TokenStore } from "./store.js";
10
+ /** The knobs of {@link classify}. */
11
+ export interface ClassifyOptions {
12
+ /** How many documents of weight the prior carries against a token's own counts. */
13
+ strength: number;
14
+ /** The probability assumed for a token never seen. */
15
+ prior: number;
16
+ /** The most tokens that enter the combination. */
17
+ interestingTokens: number;
18
+ /** How far from 0.5 a token's probability must sit to enter the combination. */
19
+ minDeviation: number;
20
+ }
21
+ /** One token that entered the combination. */
22
+ export interface Clue {
23
+ token: string;
24
+ probability: number;
25
+ }
26
+ /** The combined verdict and the tokens behind it, most telling first. */
27
+ export interface Classification {
28
+ probability: number;
29
+ clues: Clue[];
30
+ }
31
+ /**
32
+ * Combines the most telling of `tokens` into a spam probability. A token's probability is the
33
+ * ratio of its spam and ham document frequencies, pulled toward `prior` in proportion to how
34
+ * rarely it was seen. With no telling token the answer is 0.5, which no cut acts on.
35
+ *
36
+ * @param tokens - The submission's distinct tokens
37
+ * @param counts - What the store holds for them
38
+ */
39
+ export declare function classify(tokens: readonly string[], counts: TokenStore.Counts, options: ClassifyOptions): Classification;
@@ -0,0 +1,72 @@
1
+ /**
2
+ * The distance from 0 and 1 a token probability is held at before taking logarithms, so a
3
+ * `strength` of 0 with a token seen only in one label yields a finite combination.
4
+ */
5
+ const PROBABILITY_FLOOR = 1e-6;
6
+ /**
7
+ * Combines the most telling of `tokens` into a spam probability. A token's probability is the
8
+ * ratio of its spam and ham document frequencies, pulled toward `prior` in proportion to how
9
+ * rarely it was seen. With no telling token the answer is 0.5, which no cut acts on.
10
+ *
11
+ * @param tokens - The submission's distinct tokens
12
+ * @param counts - What the store holds for them
13
+ */
14
+ export function classify(tokens, counts, options) {
15
+ let clues = [];
16
+ for (let token of tokens) {
17
+ let probability = tokenProbability(counts.tokens.get(token), counts.documents, options);
18
+ if (Math.abs(probability - 0.5) >= options.minDeviation)
19
+ clues.push({ token, probability });
20
+ }
21
+ clues.sort((a, b) => Math.abs(b.probability - 0.5) - Math.abs(a.probability - 0.5));
22
+ clues = clues.slice(0, options.interestingTokens);
23
+ return { probability: fisher(clues.map((clue) => clue.probability)), clues };
24
+ }
25
+ /**
26
+ * Robinson's f(w) = (s·x + n·p(w)) / (s + n), where p(w) compares the token's frequency among
27
+ * spam documents with its frequency among ham documents and n is how many documents held it.
28
+ */
29
+ function tokenProbability(tally, documents, options) {
30
+ let spam = tally?.spam ?? 0;
31
+ let ham = tally?.ham ?? 0;
32
+ let seen = spam + ham;
33
+ if (seen === 0)
34
+ return options.prior;
35
+ let spamFrequency = documents.spam > 0 ? Math.min(spam / documents.spam, 1) : 0;
36
+ let hamFrequency = documents.ham > 0 ? Math.min(ham / documents.ham, 1) : 0;
37
+ let ratio = spamFrequency / (spamFrequency + hamFrequency);
38
+ return (options.strength * options.prior + seen * ratio) / (options.strength + seen);
39
+ }
40
+ /**
41
+ * Fisher's method as SpamBayes applies it: one chi-squared test for "these tokens are spammy"
42
+ * and one for "these tokens are hammy", averaged so disagreement lands near 0.5.
43
+ */
44
+ function fisher(probabilities) {
45
+ if (probabilities.length === 0)
46
+ return 0.5;
47
+ let spamLog = 0;
48
+ let hamLog = 0;
49
+ for (let raw of probabilities) {
50
+ let probability = Math.min(Math.max(raw, PROBABILITY_FLOOR), 1 - PROBABILITY_FLOOR);
51
+ spamLog += Math.log(1 - probability);
52
+ hamLog += Math.log(probability);
53
+ }
54
+ let freedom = 2 * probabilities.length;
55
+ let spamminess = 1 - chiSquaredSurvival(-2 * spamLog, freedom);
56
+ let hamminess = 1 - chiSquaredSurvival(-2 * hamLog, freedom);
57
+ return (1 + spamminess - hamminess) / 2;
58
+ }
59
+ /**
60
+ * The probability that a chi-squared variable with an even `freedom` exceeds `value`, by the
61
+ * closed-form series for even degrees of freedom, clamped to 1 against rounding.
62
+ */
63
+ function chiSquaredSurvival(value, freedom) {
64
+ let half = value / 2;
65
+ let term = Math.exp(-half);
66
+ let sum = term;
67
+ for (let index = 1; index < freedom / 2; index++) {
68
+ term *= half / index;
69
+ sum += term;
70
+ }
71
+ return Math.min(sum, 1);
72
+ }
@@ -0,0 +1,58 @@
1
+ /**
2
+ * A token store over `remix/data-table`, for D1 and Durable Object SQLite alike. Every write is
3
+ * an `insert … on conflict do update` adding to the stored counts, so concurrent reports never
4
+ * lose an increment without the transactions D1 lacks. The host app owns the table's migration.
5
+ *
6
+ * @author [Sergio Xalambrí](https://sergiodxa.com)
7
+ * @copyright Sergio Xalambrí 2026
8
+ */
9
+ import type { Result } from "@sdxc/result";
10
+ import type { Database, TableRow } from "remix/data-table";
11
+ import type { Label } from "../check.js";
12
+ import { SpamCheckError } from "../check.js";
13
+ import type { TokenStore } from "./store.js";
14
+ /**
15
+ * One row per token with the number of spam and ham documents it appeared in. The row for
16
+ * {@link DOCUMENTS_TOKEN} holds the document totals. See {@link SPAM_TOKENS_SCHEMA_SQL}.
17
+ */
18
+ export declare const spamTokens: import("remix/data-table").Table<"spam_tokens", {
19
+ token: import("remix/data-table").ColumnBuilder<string>;
20
+ spam: import("remix/data-table").ColumnBuilder<number>;
21
+ ham: import("remix/data-table").ColumnBuilder<number>;
22
+ }, readonly "token"[]>;
23
+ /** A token's counts, as loaded from {@link spamTokens}. */
24
+ export type SpamTokenRow = TableRow<typeof spamTokens>;
25
+ /**
26
+ * SQL creating the table {@link DataTableTokenStore} needs, for a host app to paste into its own
27
+ * migration. The primary key is the only index, since every read and write is by token.
28
+ */
29
+ export declare const SPAM_TOKENS_SCHEMA_SQL = "create table spam_tokens (\n\ttoken text primary key,\n\tspam integer not null default 0,\n\tham integer not null default 0\n);";
30
+ /**
31
+ * The row holding the document totals. Tokenized content never contains `@` outside a prefixed
32
+ * token, so this key never collides with a real token.
33
+ */
34
+ export declare const DOCUMENTS_TOKEN = "@documents";
35
+ /**
36
+ * Stores counts in {@link spamTokens}. A report writes its tokens in statements of at most
37
+ * {@link TOKENS_PER_STATEMENT}, with the document total in the last one, so a failure part way
38
+ * undercounts a document's tokens and never counts a document whose tokens were not written.
39
+ */
40
+ export declare class DataTableTokenStore implements TokenStore {
41
+ #private;
42
+ /**
43
+ * @param db - A database whose schema includes {@link spamTokens}; one per tenant
44
+ */
45
+ constructor(db: Database);
46
+ /**
47
+ * Reads the requested tokens and the totals row in batches of {@link TOKENS_PER_STATEMENT}.
48
+ *
49
+ * @returns The counts, or an `unavailable` error when a query fails
50
+ */
51
+ read(tokens: readonly string[]): Promise<Result<TokenStore.Counts, SpamCheckError>>;
52
+ /**
53
+ * Adds one document with single-statement upserts; see {@link DataTableTokenStore}.
54
+ *
55
+ * @returns Success, or an `unavailable` error when a write fails
56
+ */
57
+ increment(tokens: readonly string[], label: Label): Promise<Result<void, SpamCheckError>>;
58
+ }
@@ -0,0 +1,123 @@
1
+ import { failure, success } from "@sdxc/result";
2
+ import { column as c, inList, rawSql, table } from "remix/data-table";
3
+ import { SpamCheckError } from "../check.js";
4
+ /**
5
+ * One row per token with the number of spam and ham documents it appeared in. The row for
6
+ * {@link DOCUMENTS_TOKEN} holds the document totals. See {@link SPAM_TOKENS_SCHEMA_SQL}.
7
+ */
8
+ export const spamTokens = table({
9
+ name: "spam_tokens",
10
+ primaryKey: ["token"],
11
+ columns: {
12
+ token: c.text(),
13
+ spam: c.integer(),
14
+ ham: c.integer(),
15
+ },
16
+ });
17
+ /**
18
+ * SQL creating the table {@link DataTableTokenStore} needs, for a host app to paste into its own
19
+ * migration. The primary key is the only index, since every read and write is by token.
20
+ */
21
+ export const SPAM_TOKENS_SCHEMA_SQL = `create table spam_tokens (
22
+ token text primary key,
23
+ spam integer not null default 0,
24
+ ham integer not null default 0
25
+ );`;
26
+ /**
27
+ * The row holding the document totals. Tokenized content never contains `@` outside a prefixed
28
+ * token, so this key never collides with a real token.
29
+ */
30
+ export const DOCUMENTS_TOKEN = "@documents";
31
+ /** Tokens bound per statement, keeping each query under D1's limit of 100 bound parameters. */
32
+ const TOKENS_PER_STATEMENT = 90;
33
+ /**
34
+ * Stores counts in {@link spamTokens}. A report writes its tokens in statements of at most
35
+ * {@link TOKENS_PER_STATEMENT}, with the document total in the last one, so a failure part way
36
+ * undercounts a document's tokens and never counts a document whose tokens were not written.
37
+ */
38
+ export class DataTableTokenStore {
39
+ #db;
40
+ /**
41
+ * @param db - A database whose schema includes {@link spamTokens}; one per tenant
42
+ */
43
+ constructor(db) {
44
+ this.#db = db;
45
+ }
46
+ /**
47
+ * Reads the requested tokens and the totals row in batches of {@link TOKENS_PER_STATEMENT}.
48
+ *
49
+ * @returns The counts, or an `unavailable` error when a query fails
50
+ */
51
+ async read(tokens) {
52
+ let keys = [DOCUMENTS_TOKEN, ...new Set(tokens)];
53
+ let rows = [];
54
+ try {
55
+ for (let batch of chunk(keys, TOKENS_PER_STATEMENT)) {
56
+ rows.push(...(await this.#db.findMany(spamTokens, { where: inList("token", batch) })));
57
+ }
58
+ }
59
+ catch (error) {
60
+ return failure(storeError("could not be read", error));
61
+ }
62
+ let documents = { spam: 0, ham: 0 };
63
+ let found = new Map();
64
+ for (let row of rows) {
65
+ let tally = { spam: storedCount(row.spam), ham: storedCount(row.ham) };
66
+ if (row.token === DOCUMENTS_TOKEN)
67
+ documents = tally;
68
+ else
69
+ found.set(row.token, tally);
70
+ }
71
+ return success({ documents, tokens: found });
72
+ }
73
+ /**
74
+ * Adds one document with single-statement upserts; see {@link DataTableTokenStore}.
75
+ *
76
+ * @returns Success, or an `unavailable` error when a write fails
77
+ */
78
+ async increment(tokens, label) {
79
+ let batches = chunk([...new Set(tokens)], TOKENS_PER_STATEMENT);
80
+ if (batches.length === 0)
81
+ batches.push([]);
82
+ let row = label === "spam" ? "(?, 1, 0)" : "(?, 0, 1)";
83
+ try {
84
+ for (let [index, batch] of batches.entries()) {
85
+ let keys = index === batches.length - 1 ? [...batch, DOCUMENTS_TOKEN] : batch;
86
+ await this.#db.exec(rawSql(`insert into spam_tokens (token, spam, ham) values ${keys.map(() => row).join(", ")}
87
+ on conflict (token) do update set
88
+ spam = spam_tokens.spam + excluded.spam,
89
+ ham = spam_tokens.ham + excluded.ham`, keys));
90
+ }
91
+ }
92
+ catch (error) {
93
+ return failure(storeError("could not be written", error));
94
+ }
95
+ return success(undefined);
96
+ }
97
+ }
98
+ /**
99
+ * Splits `items` into consecutive batches of at most `size`.
100
+ *
101
+ * @template T - The item type
102
+ * @returns No batches for an empty list
103
+ */
104
+ function chunk(items, size) {
105
+ let batches = [];
106
+ for (let start = 0; start < items.length; start += size) {
107
+ batches.push(items.slice(start, start + size));
108
+ }
109
+ return batches;
110
+ }
111
+ /**
112
+ * Coerces a stored count that a driver may hand back as text or a bigint, reading anything
113
+ * unusable as zero so one bad row leaves the rest of the classification intact.
114
+ */
115
+ function storedCount(value) {
116
+ let count = Number(value);
117
+ return Number.isFinite(count) && count > 0 ? Math.trunc(count) : 0;
118
+ }
119
+ /** Wraps a driver error as the `unavailable` failure the filter records and moves past. */
120
+ function storeError(what, cause) {
121
+ let reason = cause instanceof Error ? cause.message : String(cause);
122
+ return new SpamCheckError("unavailable", `the spam token table ${what}: ${reason}`);
123
+ }
@@ -0,0 +1,18 @@
1
+ /**
2
+ * An in-process token store for tests and local development. It is the reference for the counts
3
+ * a persistent store keeps, with state that lives and dies with the instance.
4
+ *
5
+ * @author [Sergio Xalambrí](https://sergiodxa.com)
6
+ * @copyright Sergio Xalambrí 2026
7
+ */
8
+ import type { Result } from "@sdxc/result";
9
+ import type { Label, SpamCheckError } from "../check.js";
10
+ import type { TokenStore } from "./store.js";
11
+ /** Keeps counts in a `Map` owned by the instance, so two instances never share training. */
12
+ export declare class MemoryTokenStore implements TokenStore {
13
+ #private;
14
+ /** Answers copies, so a caller mutating the result leaves the stored counts intact. */
15
+ read(tokens: readonly string[]): Promise<Result<TokenStore.Counts, SpamCheckError>>;
16
+ /** Counts each distinct token once, however often `tokens` repeats it. */
17
+ increment(tokens: readonly string[], label: Label): Promise<Result<void, SpamCheckError>>;
18
+ }
@@ -0,0 +1,26 @@
1
+ import { success } from "@sdxc/result";
2
+ /** Keeps counts in a `Map` owned by the instance, so two instances never share training. */
3
+ export class MemoryTokenStore {
4
+ #documents = { spam: 0, ham: 0 };
5
+ #tokens = new Map();
6
+ /** Answers copies, so a caller mutating the result leaves the stored counts intact. */
7
+ async read(tokens) {
8
+ let found = new Map();
9
+ for (let token of tokens) {
10
+ let tally = this.#tokens.get(token);
11
+ if (tally !== undefined)
12
+ found.set(token, { ...tally });
13
+ }
14
+ return success({ documents: { ...this.#documents }, tokens: found });
15
+ }
16
+ /** Counts each distinct token once, however often `tokens` repeats it. */
17
+ async increment(tokens, label) {
18
+ for (let token of new Set(tokens)) {
19
+ let tally = this.#tokens.get(token) ?? { spam: 0, ham: 0 };
20
+ tally[label] += 1;
21
+ this.#tokens.set(token, tally);
22
+ }
23
+ this.#documents[label] += 1;
24
+ return success(undefined);
25
+ }
26
+ }
@@ -0,0 +1,41 @@
1
+ /**
2
+ * The storage contract behind the classifier: per-token counts of the spam and ham documents a
3
+ * token appeared in, plus how many documents of each label were reported. Keeping it to one read
4
+ * and one increment lets any backend implement it with single statements.
5
+ *
6
+ * @author [Sergio Xalambrí](https://sergiodxa.com)
7
+ * @copyright Sergio Xalambrí 2026
8
+ */
9
+ import type { Result } from "@sdxc/result";
10
+ import type { Label, SpamCheckError } from "../check.js";
11
+ /**
12
+ * Where the classifier keeps what moderators taught it. A multi-tenant app builds one store per
13
+ * tenant database, so each tenant's reports shape only that tenant's verdicts.
14
+ */
15
+ export interface TokenStore {
16
+ /**
17
+ * Loads the counts for `tokens` and the document totals. A token never reported is absent
18
+ * from the returned map, which the classifier reads as zero of each label.
19
+ */
20
+ read(tokens: readonly string[]): Promise<Result<TokenStore.Counts, SpamCheckError>>;
21
+ /**
22
+ * Adds one `label` document: every token's count for that label and the label's document
23
+ * total each rise by one. Callers pass distinct tokens.
24
+ */
25
+ increment(tokens: readonly string[], label: Label): Promise<Result<void, SpamCheckError>>;
26
+ }
27
+ /** The types a {@link TokenStore} returns. */
28
+ export declare namespace TokenStore {
29
+ /** How many documents of each label an entry covers. */
30
+ interface Tally {
31
+ spam: number;
32
+ ham: number;
33
+ }
34
+ /** The answer to {@link TokenStore.read}. */
35
+ interface Counts {
36
+ /** Every document ever reported, by label. */
37
+ documents: Tally;
38
+ /** The requested tokens that have been reported at least once. */
39
+ tokens: Map<string, Tally>;
40
+ }
41
+ }
File without changes
@@ -0,0 +1,31 @@
1
+ /**
2
+ * Turns a submission into the tokens the classifier counts: content words, linked hosts and the
3
+ * author's email domain. Classifying and training share this one function, so a token learned
4
+ * from a report is spelled exactly as the check later looks it up.
5
+ *
6
+ * @author [Sergio Xalambrí](https://sergiodxa.com)
7
+ * @copyright Sergio Xalambrí 2026
8
+ */
9
+ import type { Submission } from "../check.js";
10
+ /**
11
+ * The distinct tokens of `submission`, hosts first, then the email domain, then words in order of
12
+ * appearance, up to `maxTokens`. Hosts drop a leading `www.` and carry the `host:` prefix, the
13
+ * domain carries `email-domain:`, and words are lowercased and kept within the length window.
14
+ *
15
+ * @example tokenize({ content: "Buy at https://www.shop.example" }); // ["host:shop.example", "buy"]
16
+ */
17
+ export declare function tokenize(submission: Submission, options?: tokenize.Options): string[];
18
+ /** The types {@link tokenize} reads. */
19
+ export declare namespace tokenize {
20
+ /** How content becomes tokens; the classifier and its training must share these values. */
21
+ interface Options {
22
+ /** Shorter words carry no meaning worth counting. @default 3 */
23
+ minLength?: number;
24
+ /** Longer words are hashes, encodings or keyboard mashing. @default 24 */
25
+ maxLength?: number;
26
+ /** Bounds the store reads and writes a single submission can cause. @default 200 */
27
+ maxTokens?: number;
28
+ /** Counts the author's email domain as an `email-domain:` token. @default true */
29
+ emailDomain?: boolean;
30
+ }
31
+ }
@@ -0,0 +1,42 @@
1
+ import { extractLinks, extractWords } from "../lib/text.js";
2
+ /** Apostrophes a word starts or ends with, which quote the word rather than belong to it. */
3
+ const EDGE_APOSTROPHES = /^['’]+|['’]+$/g;
4
+ /**
5
+ * The distinct tokens of `submission`, hosts first, then the email domain, then words in order of
6
+ * appearance, up to `maxTokens`. Hosts drop a leading `www.` and carry the `host:` prefix, the
7
+ * domain carries `email-domain:`, and words are lowercased and kept within the length window.
8
+ *
9
+ * @example tokenize({ content: "Buy at https://www.shop.example" }); // ["host:shop.example", "buy"]
10
+ */
11
+ export function tokenize(submission, options = {}) {
12
+ let minLength = options.minLength ?? 3;
13
+ let maxLength = options.maxLength ?? 24;
14
+ let maxTokens = options.maxTokens ?? 200;
15
+ let emailDomain = options.emailDomain ?? true;
16
+ let tokens = new Set();
17
+ let urls = extractLinks(submission.content);
18
+ let authorUrl = submission.author?.url;
19
+ if (authorUrl !== undefined && URL.canParse(authorUrl))
20
+ urls.push(new URL(authorUrl));
21
+ for (let url of urls) {
22
+ let host = url.hostname.toLowerCase().replace(/^www\./, "");
23
+ if (host.length > 0)
24
+ tokens.add(`host:${host}`);
25
+ }
26
+ let email = submission.author?.email;
27
+ if (emailDomain && email !== undefined) {
28
+ let domain = email
29
+ .slice(email.lastIndexOf("@") + 1)
30
+ .trim()
31
+ .toLowerCase();
32
+ if (email.includes("@") && domain.length > 0)
33
+ tokens.add(`email-domain:${domain}`);
34
+ }
35
+ for (let raw of extractWords(submission.content)) {
36
+ let word = raw.replace(EDGE_APOSTROPHES, "").toLowerCase();
37
+ let length = Array.from(word).length;
38
+ if (length >= minLength && length <= maxLength)
39
+ tokens.add(word);
40
+ }
41
+ return [...tokens].slice(0, maxTokens);
42
+ }
@@ -0,0 +1,43 @@
1
+ import type { TokenStore } from "./bayes/store.js";
2
+ import type { SpamCheck } from "./check.js";
3
+ import { tokenize } from "./bayes/tokens.js";
4
+ export type { SpamTokenRow } from "./bayes/data-table.js";
5
+ export type { TokenStore } from "./bayes/store.js";
6
+ export { DataTableTokenStore, SPAM_TOKENS_SCHEMA_SQL, spamTokens } from "./bayes/data-table.js";
7
+ export { MemoryTokenStore } from "./bayes/memory.js";
8
+ export { tokenize } from "./bayes/tokens.js";
9
+ /**
10
+ * Scores a submission by what moderators reported before. It runs in the `local` stage because
11
+ * it reads the app's own storage, never a third party, and that read is its only await. Until the
12
+ * store holds `minDocuments` reports of each label it emits nothing, so a fresh install relies on
13
+ * the rules alone. `report` trains it; build it per tenant from that tenant's store.
14
+ *
15
+ * @example createSpamFilter({ checks: [...DEFAULT_RULES, bayes({ store: new DataTableTokenStore(ctx.db) })] })
16
+ */
17
+ export declare function bayes(options: bayes.Options): SpamCheck;
18
+ /** The types {@link bayes} reads. */
19
+ export declare namespace bayes {
20
+ /** Where counts live, when the classifier speaks, and how much its verdicts weigh. */
21
+ interface Options extends tokenize.Options {
22
+ /** One per tenant in a multi-tenant app, so training never crosses tenants. */
23
+ store: TokenStore;
24
+ /** Reports of each label required before any signal. @default 20 */
25
+ minDocuments?: number;
26
+ /** The spam probability at or above which `bayes.spam` is emitted. @default 0.9 */
27
+ spamCut?: number;
28
+ /** The spam probability at or below which `bayes.ham` is emitted. @default 0.2 */
29
+ hamCut?: number;
30
+ /** @default 6 */
31
+ spamScore?: number;
32
+ /** Negative, so a submission the site's history calls ham is pulled down. @default -4 */
33
+ hamScore?: number;
34
+ /** How many documents of weight the prior carries against a rarely seen token. @default 1 */
35
+ strength?: number;
36
+ /** The probability assumed for a token never seen. @default 0.5 */
37
+ prior?: number;
38
+ /** The most telling tokens that enter the combination. @default 15 */
39
+ interestingTokens?: number;
40
+ /** How far from 0.5 a token must sit to enter the combination. @default 0.1 */
41
+ minDeviation?: number;
42
+ }
43
+ }
package/dist/bayes.js ADDED
@@ -0,0 +1,67 @@
1
+ import { isFailure, success } from "@sdxc/result";
2
+ import { classify } from "./bayes/classify.js";
3
+ import { tokenize } from "./bayes/tokens.js";
4
+ export { DataTableTokenStore, SPAM_TOKENS_SCHEMA_SQL, spamTokens } from "./bayes/data-table.js";
5
+ export { MemoryTokenStore } from "./bayes/memory.js";
6
+ export { tokenize } from "./bayes/tokens.js";
7
+ /** How many of the most telling tokens a signal's detail names. */
8
+ const DETAIL_TOKENS = 5;
9
+ /**
10
+ * Scores a submission by what moderators reported before. It runs in the `local` stage because
11
+ * it reads the app's own storage, never a third party, and that read is its only await. Until the
12
+ * store holds `minDocuments` reports of each label it emits nothing, so a fresh install relies on
13
+ * the rules alone. `report` trains it; build it per tenant from that tenant's store.
14
+ *
15
+ * @example createSpamFilter({ checks: [...DEFAULT_RULES, bayes({ store: new DataTableTokenStore(ctx.db) })] })
16
+ */
17
+ export function bayes(options) {
18
+ let store = options.store;
19
+ let minDocuments = options.minDocuments ?? 20;
20
+ let spamCut = options.spamCut ?? 0.9;
21
+ let hamCut = options.hamCut ?? 0.2;
22
+ let spamScore = options.spamScore ?? 6;
23
+ let hamScore = options.hamScore ?? -4;
24
+ let classifyOptions = {
25
+ strength: options.strength ?? 1,
26
+ prior: options.prior ?? 0.5,
27
+ interestingTokens: options.interestingTokens ?? 15,
28
+ minDeviation: options.minDeviation ?? 0.1,
29
+ };
30
+ return {
31
+ name: "bayes",
32
+ stage: "local",
33
+ /**
34
+ * Emits `bayes.spam` above `spamCut`, `bayes.ham` below `hamCut`, and nothing between them
35
+ * or before the cold-start gate opens. A failed store read is a failure, never a verdict.
36
+ */
37
+ async check(submission) {
38
+ let tokens = tokenize(submission, options);
39
+ let read = await store.read(tokens);
40
+ if (isFailure(read))
41
+ return read;
42
+ let counts = read.data;
43
+ if (counts.documents.spam < minDocuments || counts.documents.ham < minDocuments) {
44
+ return success([]);
45
+ }
46
+ let { probability, clues } = classify(tokens, counts, classifyOptions);
47
+ let isSpam = probability >= spamCut;
48
+ if (!isSpam && probability > hamCut)
49
+ return success([]);
50
+ let named = clues
51
+ .filter((clue) => (isSpam ? clue.probability > 0.5 : clue.probability < 0.5))
52
+ .slice(0, DETAIL_TOKENS)
53
+ .map((clue) => clue.token);
54
+ return success([
55
+ {
56
+ check: isSpam ? "bayes.spam" : "bayes.ham",
57
+ score: isSpam ? spamScore : hamScore,
58
+ detail: `spam probability ${probability.toFixed(3)}; tokens: ${named.join(", ")}`,
59
+ },
60
+ ]);
61
+ },
62
+ /** Counts the submission's tokens under `label`, tokenized exactly as `check` reads them. */
63
+ async report(submission, label) {
64
+ return await store.increment(tokenize(submission, options), label);
65
+ },
66
+ };
67
+ }