@pseolint/core 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -4,20 +4,32 @@
|
|
|
4
4
|
* Loads the prebuilt Wikipedia trigram bloom filter and provides a
|
|
5
5
|
* paraphrase-rate estimator. Used by content/wikipedia-paraphrase rule.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
7
|
+
* v0.6.2 — bloom filter inlined as base64 to remove filesystem dependency.
|
|
8
|
+
* Production hotfix: Vercel serverless deployments couldn't resolve the
|
|
9
|
+
* relative file path. Inlining the 8 KB binary into the JS source removes
|
|
10
|
+
* deployment-config dependencies and makes the rule work in any runtime
|
|
11
|
+
* (Node, Bun, Cloudflare Workers, Vercel serverless, etc.).
|
|
12
|
+
*
|
|
13
|
+
* Bloom filter layout:
|
|
8
14
|
* m = 65536 bits (8192 bytes)
|
|
9
15
|
* k = 3 FNV-1a-32 hash functions with distinct seeds
|
|
10
16
|
*
|
|
11
17
|
* FP rate ~5% for the curated corpus of ~10 k unique trigrams.
|
|
18
|
+
*
|
|
19
|
+
* If the underlying corpus is regenerated via `bun run build-wikipedia-bloom`,
|
|
20
|
+
* also re-run the inline script that produced BLOOM_BASE64 below — see
|
|
21
|
+
* `scripts/inline-wikipedia-bloom.ts` (or regenerate manually):
|
|
22
|
+
* node -e "console.log(require(node:fs).readFileSync(packages/core/data/wikipedia-trigrams.bin).toString(base64))"
|
|
12
23
|
*/
|
|
13
|
-
/**
|
|
24
|
+
/** Decode and cache the bloom filter from the inlined base64 string. */
|
|
14
25
|
export declare function loadWikipediaBloomFilter(): Uint8Array;
|
|
15
26
|
/**
|
|
16
|
-
* Compute the
|
|
17
|
-
*
|
|
18
|
-
*
|
|
27
|
+
* Compute the paraphrase rate of `text` against the Wikipedia reference
|
|
28
|
+
* corpus. Returns a value in [0, 1] — the fraction of trigrams that are
|
|
29
|
+
* present in the bloom filter. Returns 0 for text shorter than 3 tokens.
|
|
19
30
|
*/
|
|
20
31
|
export declare function wikipediaParaphraseRate(text: string): number;
|
|
21
|
-
/**
|
|
32
|
+
/** Test-only: reset the bloom-filter cache so loadWikipediaBloomFilter
|
|
33
|
+
* re-decodes the inlined base64 on next call. Used by unit tests. */
|
|
22
34
|
export declare function _resetBloomCache(): void;
|
|
23
35
|
//# sourceMappingURL=wikipedia-paraphrase.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wikipedia-paraphrase.d.ts","sourceRoot":"","sources":["../../src/algorithms/wikipedia-paraphrase.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"wikipedia-paraphrase.d.ts","sourceRoot":"","sources":["../../src/algorithms/wikipedia-paraphrase.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAoBH,wEAAwE;AACxE,wBAAgB,wBAAwB,IAAI,UAAU,CAUrD;AAuBD;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAS5D;AAED;sEACsE;AACtE,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}
|
|
@@ -4,17 +4,24 @@
|
|
|
4
4
|
* Loads the prebuilt Wikipedia trigram bloom filter and provides a
|
|
5
5
|
* paraphrase-rate estimator. Used by content/wikipedia-paraphrase rule.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
7
|
+
* v0.6.2 — bloom filter inlined as base64 to remove filesystem dependency.
|
|
8
|
+
* Production hotfix: Vercel serverless deployments couldn't resolve the
|
|
9
|
+
* relative file path. Inlining the 8 KB binary into the JS source removes
|
|
10
|
+
* deployment-config dependencies and makes the rule work in any runtime
|
|
11
|
+
* (Node, Bun, Cloudflare Workers, Vercel serverless, etc.).
|
|
12
|
+
*
|
|
13
|
+
* Bloom filter layout:
|
|
8
14
|
* m = 65536 bits (8192 bytes)
|
|
9
15
|
* k = 3 FNV-1a-32 hash functions with distinct seeds
|
|
10
16
|
*
|
|
11
17
|
* FP rate ~5% for the curated corpus of ~10 k unique trigrams.
|
|
18
|
+
*
|
|
19
|
+
* If the underlying corpus is regenerated via `bun run build-wikipedia-bloom`,
|
|
20
|
+
* also re-run the inline script that produced BLOOM_BASE64 below — see
|
|
21
|
+
* `scripts/inline-wikipedia-bloom.ts` (or regenerate manually):
|
|
22
|
+
* node -e "console.log(require(node:fs).readFileSync(packages/core/data/wikipedia-trigrams.bin).toString(base64))"
|
|
12
23
|
*/
|
|
13
|
-
import { readFileSync } from "node:fs";
|
|
14
|
-
import { join, dirname } from "node:path";
|
|
15
|
-
import { fileURLToPath } from "node:url";
|
|
16
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
17
|
-
const DATA_PATH = join(__dirname, "..", "..", "data", "wikipedia-trigrams.bin");
|
|
24
|
+
const BLOOM_BASE64 = "GgIDUAggMFAQKEEAQDKChEQAEhgFAABIQWSAAE0CAoAGWY5IAQBABMACNAAHQwAwFAE0AIABIBKoQACJoQEAANgAgwSCgThYITUELRZoACwAIAyAgXAA0BQSKQABVAGBBcYKQmAQgkAWJQAAejgAAgEFARAAUAABkyIQAgAQEoEIABEAgBCEBQQgAQDhAQiQgBBATBCQECFBAAEQiAIKQQICE0BIAJARgAEMBALEAQkABGgDIgwAEACoJAQAUjEAB8IAIAAkAIAgiUAgAAQEAiECEUByAQNHEAISAiSAAAigAC5SEICACgFECAQBQFAAIQAKEhkAkFgEEIMIUAAFIZAEgpCZcAoAEJEwhKIDNDIALAQwUCSRIjABFQATGAg6gAJUEACAQBaAEAgEBAAAGAAKaoAkJRIgihAWIAiAgEIBAZUgaEASgwAQBoBIMSIBAAgSDgAMXiBEYCgABEoGAGiKSQICmhADgERCBRBCAQcYCACCBBCNDgAGAQGBKIBGABDCCIBAFEQogCSgkACAgcGoVBBAYEEC0gQBCaAAAAAgAClGQCASKIIDaBAQERCAkBCASRJiA4IwQKEAECyIAeQRwQFAPCAJKUDAABIMQAAQKAgQiAgUIMZAOAEICCAoEJIGEBpCEAiAArBIhgMAUCApIAAgAlQAIAiUikRQABAAEIBgAiAcQ4gAkCAAAymENQATAgQgEiUiOCwQAAGCIAKQlCQ3IARICAAggAwEUAAaAIhAABdI+OEaiYEAQjJQAEAQAChAGIYDqKQECAogQBCMJEAFFgARikRQAEKiCgEQQ2AIFJxAIFACAAAQAiAiqAMIACABgBIBIMUEBhBwDgDpRQSSIC8EIEniACAKhCYAAkAAAIB5BAARiFAQiACoKypjIggJCiS0GCCIQAmBAYgCU50EABAAKYMCIIMEGAgATCgAgAQphAAaAEwjKFAABRAgAEiADA+KKACDBAQIAiICwABAxCCTAggQABQCiADCYAABABAAAYHiMQAKCAU8ImkEDIoCAAEAEIQKgDZcaJRAUACRllygxQASIABADAAigYgkCEAhAASIBCyKAqCKINAADYVAkQEgaYgIIwkiIUBEYCgQBCAgAKAgEVhKQgMAABgKoRpBiAAAkoSChQCgAERiAgiEAEGJQAhAVBAIRCsFAIESYGQFAKgQIjAgUAASABEQwABBDkQhwKoIglwlFQQRAAABFgQAoAAkoYAABIIAAgIEgwgIgAACoIiC0DQoVgCQyoABAECBEiEQCBgMPAIAAJBQdwxAAAAEGREgAAEYAoABBMADCYIIABCgEAoACgUqBBaAgiIAOwEIEwAoCQCQINDAChBgQAZAAAgyigYUEAShIpABXyAYBAoUBBBByAABCAIAoAQABSEqFQCmFWRAAkxgAKgCoRQJgBaADACACACACkAkhktBiBgIBwCIkAAAIQkpAQQQUAGhaMESBFQiFQBgARAAsGEqEIHcAAQgGRgEACAhxAAAakCwEKQWjQBZCKAogCIAhAlAggAEBNwGRZnMdghAACaUCAAAAACQEBLCAAAAMCAJAEBCiEkEAAYgQoQAyHU1ADgASMDSAiABIYJgIIBAMUEIgcACgCAACwBEwAKMFQEEGGQBgAFgEAEBFDAAZgyIAgABAAAIAAQACIhSKEYIAEQAQgDAEEACQQOwBgmUIAEIyI4KRAATEEHhBUBIRNQgyNAMCEwBhAAiBFEAAY4AIGiAKckQCEIAAQGCBIAQCBBgAURQhQwECLAACQGoQCmiIMBAAgQIAARAgmgABELgI0gBVBACCULAAsMABCAGQECEABAaBAJQAANeQAIAcCAmBgYEAILEABIQwAEaAwaCCwGEoIwkAIAEARJAIhGCsQxHBALiHAAsCKABIAAFEAxeAKHAQAWUIAGCLkIAAAaCq5AAaEABAiEACABCAJoAAsIEAYAAAoQCShhoMwxQgEA0EBkIACkDACAClAUBACAAQACFiiIgABAAYBJAZhXmiOBABBAgoQxE1FGAAEACg0vmQYQAFAKiIAIAgAAABIEMJ6CIAAAwAhDYMUBQQEigxDCApg4/AQgYggADwAJBQGMEkF4AAQAoagxgACCkwAIAAABAIg5ECJCyB5BAAIwCDYSFAwAAkYRQEgCQgMCAQChgJRYDAGBSIhBgBIACgADGCgBGCAQDRQUhDEAwiyACIGARIhgAgZQoQZAAAEBkBAAEAKwYgAAEDAYMCRArGAAgIAJAgAdQUAEAAAJQIqQAZwNAAgCEFQYEEnDBCAEEwHACIEABIBACAIEBACiCQAQQ9AAAAEISAAhAABKCCATIQBEA1GAMBAGAgloCAQAAwCAEAWGKMAASoCCABZO1AAESJAgBABxKxAEQADoAgQAESkAAAQABACCIgjgAAAUAYKBMBAAAWAmBIAzAAAABAagAJKABIDEU2ACoApC4IARA0ABBCoBEiAsAIAgBBJkSUAAAEATPQAoFEBQAIhEAADgIoGkQAhAKNgCUZICBhABSgAAAACDlEYEAAEEAIQxAhAlCQlABAggiEgBgCQAEJIAAItEI0oIYCUBAcBQoOABDFIECKIIwIhVAhNjEDEBBIAFABSiQBOAkwAIFMAUoIiJAESECRABhAAQAEIAQMAASIAIIADAgSAATNBoAAFAGAAICIEBAACGKwBDEIQAEDEGAAFABIqBAAIQgAQAA4ACgIIEIAEAAAAAIBMYACCQUAQwAAFAggoCUgAAiISAAKAgoGAOQpaOAAABAAQElU0IAhAAxoAAAAMACiAIRKWBhQGAAAkiDRQ2JAQLgGmCgQsYARAYQVGAAAASOwYUCCAQBQhgAhcQAFAGAqiJAYQgASKmFABkAKEUKANEFBAhSGWUgICEcAJEFBEQUICDESEMhCDIAtCIBAGBkJaqGoEEAAAEAQEECACiAgRAoAUAYQCBIAoagAQREBMJABZhQEKBIzRCAAAgAAUAgDUgUCAGA1oBAAgACgiDBFBBQqkAwFGFgKSgQKEBCDNSCEAAaABAigsAIIIghQQIABsUGBAFEVKDMBgAABYAAFCkgAgAgErEgYhIEAAFoyIaKAkIECHCoEAARAAABAICiykCAsMwEIgBBRIEQAArEUCCAQyQEBJAMAUJggZAAAPQCAgAAEMABYAUIwFAgBIIEAAIgGJkSEJQAIEAQBEKgBKIEAQmCQgAklKAgm4ECghAGUCBVLJAUgCggEgAsaTYCAgWEAwBYAAiAEQAiTEoEBgkkIAIAJhCIEAAAAAoEAgABBAxQAIQACEAAEAEJQNgGYABKBAAkFwAAWBCCZECAgpIFCEgkKgQA0EBhQi4GGAQCsERIioSgRQAWCQgJCADHAmMAwACgACSYAAQhEgHUHPACIAgAAbQAIJUAQpEAgARSCoIhWACCAEBkIATABFoVgQCkQEhAEEBApgIhIADAAlCQIgAAIIAjIgAgAEIACAggEjIJlQQgGQAABiEEQAgCCEAGhAUBBQAGggUgAiAKCcGBnABqAECasegAgAggSCgYCEAA0ASCERCAjRYABAAVhQQB8lHDKHKwJIYmA0BAAgEDBAgAiMwrhBAggAxB4gAjMBECgOZAAgpiURBJAB9AEGAUAABEQggIQNAQEAEdwALCJEQFIYIARoZCBQACGAAKAJAACEBBAQRgRAF4UIABkAAADgAiQAQAEMIRhQgAGABjgCEEEAgAIBQQTBAAAAYAgAEICAIhJAE0TgmIIiQAgIAICAoDAAAFQUZABBBCUCgggAAgAABAgAAlSCCAAyMAOIzJEUYAWEAFEFEGEUADhRBmCGEbAoAGEATgAxQAADAACASQBCHAPQkCAAjAAEAQkyAgEAoAAyEQEYCSAAABMSwAIAggAFEABBEEYAgkQywAFEjCBxAAgACiBIBVAGFuAiiABgIAIQBAqgQBgUAAEIwCAAWIw5CoMAhgAAQFAS0ArKGgAIAAC8ACMIIAUCBoiDIJIACBCBCCAAhQHcASEIQRyAZABJAghECkAAgDBAAhiIGAERUAm4FAxMgEEFRBAAmgcAcAjETAYSmFCARCBYGAMIAgLCCAKCgECgAwEBKIGUjoEIkCUIABACgEEhOQAACAgAAgAAgAAiBEigkIEKIAAhiBgFFBIASoAAABIAMB3ABAiiAQCUAIHCAdAAhKIAAIgEgIYBCgYsBKAHACnhMGEDQIgEAgAAQhABwADoAIrAYIBFRAAIBEAACKUMAAAIhAgBAAWYgDgKCQBAgwED4AGACQOABZIQABAAEGAQhRgECOkgCQI0AIABAJJAxIwSQvBUggwAAYIZTUIgSAOCShBAZECAyIAhggFAIQAASBCAQABAAAAAAiiCmDBCuhYDhtOCIABAQogDAikEoQQIEJAAQUAAAwAChgAAAQAAHBgAIKEABYNUDEAFCAAAAhABgEkQ0QAEIgAAGoAAAgBJEAVQIBKGHqIwgAQBYUAiEgAqkBKQELAAHBEwAERAHApGAQBCEYIEAiIJIhAGIAsJIRRFCwhOgTBCACAQCCAAwgFBAiIAMQAIQORkIAEBCAEf4MAABAiAAGAJQEwAABnKhTCO9KACAASEBBVBIKiDEQJAADAICAApBwwBiapAdAnCBApKEAAoxIjCBEAEggQUDKgAQEADAGwGhABCnBAxAAYxCGBAAAA0IYEIuNMihiBMQhEAQQgghUBREUAIjYA2AhDERAMRIACATQBQAACFAMIwRhaiEpYkCASAOAAAEFAMhAAAcaogABAQSwBIuGKQIEEBBoAoUKAK4YGEDBoAoBGwIABACAACIGKiggEGIAAkwACkokBShISKEQCpaKAEB8UKEABgoQA4AKMBIAIDAAMCgBQhhDgQAKIAIUASkKyQkAAQwAoJjAggEBCaBABUGACCLQCEQFAIFDgRAAIcAAAhFJJAAJIIhICBAgiAKAKAkUaCJgAAKAoUABkIMEFJAAAALQCAIegMwAiCoYAQAgYhYEgIsDjYABAFIDAkEACCSCE6EIgDOcQUoIBAAAFBIB4hRACAggQIDAAAgQAoAgAQUIkRQjAAIKmASCyUEABBoAUAhyAEISUVQOUAgwAAQIAAEBAgeCGEQRSAAAEF1goBoMAAAAABkMQYeYGESAABQAaAgZAAAECQgEawAAABAQEICAVHgyIBDAIMIAEEgAcYAAEAgACAATREACgkYNFiRMoCoA4YAAZkAUAAhYEKlCCAogYGIQEACKgFSKgSIAgIAAABAgGBA3DBAzgQpYNMwRN2ABSAkkhAOEAVNEiCKIgAARAIBQAphLE4yYskVwOKwARVAGpAAXKQBACBIAAEEEIDRAaASaGgAGAFUCCCCIBSgB0AUAAAggCCgxMkQWgUIBDgAAgQBQAAMACAAACQQQEIA4gBYjAgoWBAEEAGAggACCGEiAmhAAEyMAABkACPIAgEQDFEGJkBIAsrEGkEYEiGAwIIQCcAjMAYEiBAhAQIBYYAgISKASUGAAIBgMBJAFlACAAASwACUEEYFKSIYgEAwIVaIhLTamoQAgAgCAAAGCBaFAgQc4LgIIGwkEQAAEBAhJCCAAQDAHAkQwKAakZWAIFAAKAQaBYgAQYIAWQgEgAMQEDDEAgEALIAAABGAIAgAGBKBAABIkFQIVAAECRI3FACAogCSYgwBBwGCBgYEAEAASEoKRI2kwAiADqZiMQACGABgApCAQACEihgAgBgAGIoSAARNGCACgIAgoIGAEAIAJAwIMDCGAgQDAIAKBg2ggggJbGRkAAhCAAcICCIIhMACBAMCYFQARUABgoAQwEoUBIwATIAECjDADCAB6QADAgEQAoEAAoaEDaCiAQSEECgAAAyAQAQAoRaEJMgAxEYCJMACAAAKSACCIIDhACAAwABBFAACDDpEMEgmYBgQBRCBEOQIiCCJEUAAECAKGIDgWAkQEABYAggAggAAICIQAKQQgSAGFSCgFIHABvQgEAQkBQxQIAEgEIBAQyYMFFRARESOAQAQAEEEAAZAQKOJSARIIgBWAwCaUwEKEBBJAAAAABAAQBPAgBAxqEAA4FCAAEEaBD0Ig0gAEMQBCQQBAWAIBBCBQAAmCngWAQQEABCCQAARGBAoECggAsEAFEgImko4pAAAyAHBAiAyOCwggBIIEiBDIkUQIBCABi3EqxAAQgRCjQgAsAAAAUiEEUhExIOgTgwQgJGBCEIAkAAoIAJFCkAgBBgkAOAAoQABAAqIoSKgBEIAQCAAASCAoMSUAAAwBAAIQAAkgVAAKEKAACERF0AoAECACEMaADgAEiiFRZApQEHDGEAAUIVMAwGgACAhIAhGQgQhgI2IQAIAFIBcACDAjAGAAARAiEiBEEAQAAAYMhcRQAAUGAiEMAMAAIEFBLBIBiIIwJFEmQoIBFgAIAAChASIYALEIAjAhCAAoEYDBEJEAABkAAgEQCAACAgAAANDPCAwIAIEgKAgoBAgAUDFOCkpAQEAACAEQAOgjAwTABAolEAFogAB6wAUKIABEAFEBEANgUAcBLSRMgKwCJEMIgAEAAAiAAAAIQRAZACKMEAAREICADMIAACCgBAmAbCAAAcIEDAAJkQUA0AQAAKEAAgOFgBCIBAMLLIAIgAAEDACAwAADVRAgAChAEQEqhgB4ChhQAADAIoAC4AyQMAgAASoAIiWYJUBAgQABwEIoAAABoAUAAMpSQABqStcogShQJSACQEFJjYGEnCAJMEQSACAwQEYIBAkMGBUQGQWAC0QAAGACQAAgXMtIKqgAEGMAAIAIYYklSCBAATQTEUDgAAABoAiRFAjFQgQA8QQOSAABemBAACJFAciBwFRBSAAAqgEhnoNguCADAlgAiCTQAAAEQUBIBDESgiDBAQAgBCAIAEEgIgRASYAAgkAiiAQKAAgCQiYADABqCBJQMAAEAASwABLAAEKAFgAigQAQAgAAFQiAjQgAACggLgBAgARiFBCDIAxIIAEABAAgQCAKaAAhADJBDEUEACo2AyEwSkhEJgQo5FQiCBCAjAwEiYABNEJkEqASIAA4WEQABBMAAKKARFhAKjsoIBgKQBAMAUkAIIAEEKCAVgBfAUATQQUIEgAigiUi6I4BsCQSBEJAEQhJCAowAQgzkAAAAgAwJgKKkwIAOkCLgkjQABCAAAAAQBEQABgQSk4SUBEAQQjgAAVkiGAAQCgAADAgIAECMjAogDQTCggQARDGBgICEAEXgCKBERAQKAhswBACEKAAARA1IAAAIQDcAACAUGgAFFBsH6DSCNggAtQBATCFQoIIAwTAFgAJEEGI0gJACSAAACJEhAQAgmJirICgABBjEEASggEAAQgAIEBQBBAHkgEghUwABAiAABE4QjUgCyQlAmgEAAAAAwUSCIEECAAE2+IgABeAqAgKUAEIkgEkBlABGBFIgKBSAQAUAJBQCSYhWghQAQUAAADEQEZIAAAtYIDgGAzBAEChBFAABgSECBaAgUIEBAKgNRIAkiCjQAACICxCAJoEAhECDQSARQwECBAEhEQFA4AUAOEoFABICAAAIAARSGyQgwAEAEAREBEAIQIAQIASGQ8IAIEgIIIIAgJEgABEQIAAARBaIgwUIGHIAwAARFAAIS0AgFISLAgCgAkMAJAOwQSBYDCQQkCUpABQUCEMFAwAEAChIGigBFhgsQBBEhIC/BxQGhCCAEAEAogAIKCGgACAQAkQBAECAFYBUkB4AIAAEAEEAQEAUAFUgCKIAABEYEACAKAByBYJAEACQANAANwgJAGIWBAAUEvIkrGhTQCMgiABAgBCwtIjCEGACqBQGQCJEhAAAAEBolQSGCAgJqAKQiCtCkMQQCAgAqwSAQACwAAAQACAAGCYIgiaEEAAEWMIAIwGAEAECggQABIAEAFUIBIGLAQAgACFAAaAIFQRBAhqAAAYiFhAIADAiVACAIjYhCCRwQVQEFQqKigCIKAGAABGEICBAqASkZxxMAgAEADwASUMDx0AAGTAhAAZCGEAEAIQwIAABAEwSARAAsAAAhSIIUDIQBMIwSEhYBwgUoEgADoAAtgAwDCFQAJCCBgAgEJhMggmkAYsSABYAgABQAa0EggH0YBVAABQCAAABAUIgSJEOAQEAFgAQRQgQQAgADNYgEBUKAEAgQAjAABJIAogABgKoDAgAABhAECBgAgAADkAMURDAUAoAQBAkGEAgAAKCJMAoAgYEiBQKhBBVhMJAVIAggEAQUAEAAgGAVygIEEQCATAIAWEDgGBgEAJCMFAkACQKBGxVFAQeJIwAQAQDABMDBEEggCSBLBRAQGCBBMACEgZgTTgBgkgKBBkACAAAoAgAfSQABAiQlApAAkddEAEQhgAkEiQgCFFxABgiBkBAIBE0gBg2AIECIkIAAAh0CJQAQKCTAMkJIJjgoABUhlDQCCQoDAABIECgAABgkMCKIESEAIMAGCUkgGAAACBQRKCAIgIIIxBAQQQAECEIBAAIABRFkQcAgQMY0IAAABJQgEBECBYggAOCAARAEgEIAgRhBioICCgAABSAAmkAAOBjQABCgAiBIQkkq40AAoFFAIRQJjAAAIBAQEACAFgYAKAQCgAACQJgwAguGASFCUOMIQNKoIgICAQgIAEAAAUyABmIECABgACECADAAASMohaEUIgACCAEohggMBBAIwDGAAIBCEAQTICSggAAKhFABYAYmh6CKQgBSiAQUEAAQCwAAESCzQAAAByFgggFSCEFAoAAALGARABGKBQDC0giIQVgQBxIMAoAoAYEA0FgQQIBEBQhKbAANiAhogFAIAKK4EASSaAuAAoBAILasAFCURKSIBAiAAQQUAQAARRUiAUgIIAkQgAAIDEIQjAAGFBCIzAgMYgAQCADgIlGNAQAAAKAQgoIAAAcKAWoCCBRQFCgyooEBArAABgABQAgBUQBBgSBAEAAEBjCIIQEYgSgACYCAJUzAEAQAmCAFshgQQACAQAYEJwVAaJQAARCDANEGgAAAhgBbAAJEIYmBChcEQQAhAQLAkJACAUyAgoCRoAAgQkgQBIoBSAAAAYYAAAiUACRACBoAQQQCRKIIBBCEMCwAAEIAlTAYgAgiCiJIHxEbAgAJAmmBAgDgRAEAciiAOgpEwCwKCJNCAACKAAKaAQRkjEZAqRAQFEQACIQIAFIBAAgCACAEBCkqADEJIIQBFwDACBklKQBEQBAAFACIAAnFUAQAQkAAZECCCIYCIEBAUAEYEAAYAIRAAEBACiCQQGQUEKIASMSEosCQgAESIIESCAQNAwARGgADIAAIQAmgAjSwMECICAAKkKACRgCBAgQoGAkmDAsFCAgQQoShgIJABIQAAScAAEVMjAFUgBgELA6ABIAHAkDAAggIQgEAQ4RQQzAQAiPEQMwCAAKAACAIKZJFBgAUJbQJBwfQBEiAAAIiAIANBIILwkgFAAAyKUNAMZKQuAgQZBBGGjSBBIwChqhkDpaCAQwgxCABEyUQEQgAoYJIwAAIJEyJAQKAaSCkAGAQCAgSCwAqAAAAgUICBEAMJkwFFBIXTkAGCAAIlBgAARBAiDGMsQAAKwAABAIgIABNgRISICyEBBBwIIkGYCAEAACgBIMFAADAIAIQBhAgEERIFgAgIIUBYAAsDJAJAIoggApCEGSAggAiCgQEULGgiwEMeENCADAjEwAgEAAEIKwAAQAgwoUAIBaAIgIQAKEEZgAkIQEyGBCGAAxaAAdoKwADAABCgCKUQEEAgAAhFoUACCIY0AWIGAIAjJwGFQABBgglAJmASQUcgAJhAwpCwKECIUgAUNkgADqAQIEEBgYJYNxAEgoDLOOIAQCAHAJUrAyIAEQAEEwYRHAECEAAABIAqAEpkRQQDCAwAACEUA4AlBAIQGDADRCAQAAkADAANWEAMVABAAAAAQeSAABchDAFAQEIKAgg2kB5gRERAAgACIikQABCQ8AAAjolEUMAAKwAALDyhETAAMIAICAFgFUyEBVSpEEAAQQEZQADAAQhAA0ACABOAAEIKAghwCSEQAVQAzEACNCMAgiAAABAUIgAACAAAsAYhIQCAGAMQhAhAlAgUAGQC0ggYABoAwgSZkAAAKCCgRQAEACgbCLAQc8QIAEEQBGiDAsICSAnAOEAIYI4CQKAAUSmICRAFkAAIAQKAnoKAwMEBCJABEOIKBBIkCQAAWBCsggaAICoEAoChQYAomkBCQIAACgMRDJCAMAgAAEJEAiAUEAKABoiqEJgwIAACIXADmhooQAKAQAj0CgsAAKBJEIrAQCANAAgAQFgcAAAQGAAZAgvIeEQAoxAgRBAoCCgSKkQgQKAgAGCAlQmQAYAAARIBBBoAwAEgACCooggYhIABSBBJhAACEgAghUUDLoAcAAAj5ANAAGBAlIIFgYUBAECMiIJAAAJUAAAQwJMABEkFoIAmQGpAs4hkBADAIxBAgqIAAAAgAhEoMiYFgKgAAQEAFijoxDAYpIgJCAAREQBqCAEEBACEgWKAhFCENAsgARi0NhEICCCECoKCBAABAAOACAgQgAAAUAQHAgKjAgQAIBBAAE6AICoIwgCBQFyhKQNASsQCBICAwGAwAAFgpAQaGQB9GIFgAjAEJMhCwWIAkQQygABACggEgAaEUQpNAGCIioAECVYUDBQIAmKhFAgAAKA4AFCSAwAkgDhAgEKABMQQQHAEAhSBAYExSIFgMAFgKEQICApAkUIABICBJACABCgIUAIOAgIBQLAAKYTAIoAAgiJUBAARAYAQkEGCIQGIzkErOEMKCASCCQBBiACAJBBYgVBFAcABAAigAYwAYCEjQSQAAADYgIKCAAwAEiJAcMFgkgAoCUKAEAEgKDDCHCQQDkABwRIQ+MgCACHAgEhMCmAIAcAxQAAAIMQIBFABgJAggDPEAgwgQEoBoAgA=";
|
|
18
25
|
const BLOOM_BITS = 65536;
|
|
19
26
|
const BLOOM_K = 3;
|
|
20
27
|
const FNV_PRIME = 0x01000193;
|
|
@@ -28,12 +35,17 @@ function fnv1a32(str, seed) {
|
|
|
28
35
|
return hash;
|
|
29
36
|
}
|
|
30
37
|
let _cache = null;
|
|
31
|
-
/**
|
|
38
|
+
/** Decode and cache the bloom filter from the inlined base64 string. */
|
|
32
39
|
export function loadWikipediaBloomFilter() {
|
|
33
40
|
if (_cache !== null)
|
|
34
41
|
return _cache;
|
|
35
|
-
|
|
36
|
-
|
|
42
|
+
// Buffer.from is available in Node + Bun. For browser/edge runtimes that
|
|
43
|
+
// lack it, atob + manual conversion would be the fallback (not needed yet
|
|
44
|
+
// — the engine runs server-side only).
|
|
45
|
+
const decoded = typeof Buffer !== "undefined"
|
|
46
|
+
? Buffer.from(BLOOM_BASE64, "base64")
|
|
47
|
+
: Uint8Array.from(atob(BLOOM_BASE64), (c) => c.charCodeAt(0));
|
|
48
|
+
_cache = new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength);
|
|
37
49
|
return _cache;
|
|
38
50
|
}
|
|
39
51
|
function bloomQuery(bits, trigram) {
|
|
@@ -57,25 +69,24 @@ function extractTrigrams(text) {
|
|
|
57
69
|
return trigrams;
|
|
58
70
|
}
|
|
59
71
|
/**
|
|
60
|
-
* Compute the
|
|
61
|
-
*
|
|
62
|
-
*
|
|
72
|
+
* Compute the paraphrase rate of `text` against the Wikipedia reference
|
|
73
|
+
* corpus. Returns a value in [0, 1] — the fraction of trigrams that are
|
|
74
|
+
* present in the bloom filter. Returns 0 for text shorter than 3 tokens.
|
|
63
75
|
*/
|
|
64
76
|
export function wikipediaParaphraseRate(text) {
|
|
65
|
-
if (!text || text.trim().length === 0)
|
|
66
|
-
return 0;
|
|
67
77
|
const trigrams = extractTrigrams(text);
|
|
68
78
|
if (trigrams.length === 0)
|
|
69
79
|
return 0;
|
|
70
80
|
const bits = loadWikipediaBloomFilter();
|
|
71
|
-
let
|
|
72
|
-
for (const
|
|
73
|
-
if (bloomQuery(bits,
|
|
74
|
-
|
|
81
|
+
let hits = 0;
|
|
82
|
+
for (const tg of trigrams) {
|
|
83
|
+
if (bloomQuery(bits, tg))
|
|
84
|
+
hits += 1;
|
|
75
85
|
}
|
|
76
|
-
return
|
|
86
|
+
return hits / trigrams.length;
|
|
77
87
|
}
|
|
78
|
-
/**
|
|
88
|
+
/** Test-only: reset the bloom-filter cache so loadWikipediaBloomFilter
|
|
89
|
+
* re-decodes the inlined base64 on next call. Used by unit tests. */
|
|
79
90
|
export function _resetBloomCache() {
|
|
80
91
|
_cache = null;
|
|
81
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wikipedia-paraphrase.js","sourceRoot":"","sources":["../../src/algorithms/wikipedia-paraphrase.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"wikipedia-paraphrase.js","sourceRoot":"","sources":["../../src/algorithms/wikipedia-paraphrase.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,YAAY,GAAG,8qVAA8qV,CAAC;AAEpsV,MAAM,UAAU,GAAG,KAAK,CAAC;AACzB,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,SAAS,GAAG,UAAU,CAAC;AAC7B,MAAM,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAEvD,SAAS,OAAO,CAAC,GAAW,EAAE,IAAY;IACxC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAC,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,IAAI,MAAM,GAAsB,IAAI,CAAC;AAErC,wEAAwE;AACxE,MAAM,UAAU,wBAAwB;IACtC,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IACnC,yEAAyE;IACzE,0EAA0E;IAC1E,uCAAuC;IACvC,MAAM,OAAO,GAAG,OAAO,MAAM,KAAK,WAAW;QAC3C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC;QACrC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,MAAM,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,IAAgB,EAAE,OAAe;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;IAC9D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,IAAI;SAChB,WAAW,EAAE;SACb,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC;SAClC,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,wBAAwB,EAAE,CAAC;IACxC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;YAAE,IAAI,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;AAChC,CAAC;AAED;sEACsE;AACtE,MAAM,UAAU,gBAAgB;IAC9B,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pseolint/core",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "Programmatic SEO audit engine — 32 rules across 4 categories (integrity, discoverability, citation, data) for SpamBrain risk + AI Overview citability. v0.4 verdict ladder + site classifier.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Ouranos Labs <contact@ouranos-labs.dev>",
|
|
@@ -31,8 +31,7 @@
|
|
|
31
31
|
"node": ">=18"
|
|
32
32
|
},
|
|
33
33
|
"files": [
|
|
34
|
-
"dist"
|
|
35
|
-
"data/wikipedia-trigrams.bin"
|
|
34
|
+
"dist"
|
|
36
35
|
],
|
|
37
36
|
"scripts": {
|
|
38
37
|
"build": "tsc -p tsconfig.json",
|
|
Binary file
|