@zukhruf/event-detection 0.0.0-stage → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +110 -2
  3. package/dist/analyse.d.ts +6 -0
  4. package/dist/analyse.d.ts.map +1 -0
  5. package/dist/analyse.js +26 -0
  6. package/dist/analyse.js.map +1 -0
  7. package/dist/day.d.ts +2 -0
  8. package/dist/day.d.ts.map +1 -0
  9. package/dist/day.js +3 -0
  10. package/dist/day.js.map +1 -0
  11. package/dist/design-3152b.d.ts +3 -0
  12. package/dist/design-3152b.d.ts.map +1 -0
  13. package/dist/design-3152b.js +47 -0
  14. package/dist/design-3152b.js.map +1 -0
  15. package/dist/design.d.ts +33 -0
  16. package/dist/design.d.ts.map +1 -0
  17. package/dist/design.js +2 -0
  18. package/dist/design.js.map +1 -0
  19. package/dist/event-detector.d.ts +30 -0
  20. package/dist/event-detector.d.ts.map +1 -0
  21. package/dist/event-detector.js +69 -0
  22. package/dist/event-detector.js.map +1 -0
  23. package/dist/event-table.d.ts +28 -0
  24. package/dist/event-table.d.ts.map +1 -0
  25. package/dist/event-table.js +89 -0
  26. package/dist/event-table.js.map +1 -0
  27. package/dist/grouping/assign.d.ts +5 -0
  28. package/dist/grouping/assign.d.ts.map +1 -0
  29. package/dist/grouping/assign.js +25 -0
  30. package/dist/grouping/assign.js.map +1 -0
  31. package/dist/grouping/recluster.d.ts +6 -0
  32. package/dist/grouping/recluster.d.ts.map +1 -0
  33. package/dist/grouping/recluster.js +76 -0
  34. package/dist/grouping/recluster.js.map +1 -0
  35. package/dist/grouping/scores.d.ts +11 -0
  36. package/dist/grouping/scores.d.ts.map +1 -0
  37. package/dist/grouping/scores.js +38 -0
  38. package/dist/grouping/scores.js.map +1 -0
  39. package/dist/grouping/shortlist.d.ts +10 -0
  40. package/dist/grouping/shortlist.d.ts.map +1 -0
  41. package/dist/grouping/shortlist.js +58 -0
  42. package/dist/grouping/shortlist.js.map +1 -0
  43. package/dist/index.d.ts +5 -0
  44. package/dist/index.d.ts.map +1 -0
  45. package/dist/index.js +4 -0
  46. package/dist/index.js.map +1 -0
  47. package/dist/lead.d.ts +15 -0
  48. package/dist/lead.d.ts.map +1 -0
  49. package/dist/lead.js +74 -0
  50. package/dist/lead.js.map +1 -0
  51. package/dist/numbers.d.ts +2 -0
  52. package/dist/numbers.d.ts.map +1 -0
  53. package/dist/numbers.js +19 -0
  54. package/dist/numbers.js.map +1 -0
  55. package/dist/pair-score.d.ts +11 -0
  56. package/dist/pair-score.d.ts.map +1 -0
  57. package/dist/pair-score.js +39 -0
  58. package/dist/pair-score.js.map +1 -0
  59. package/dist/schema.d.ts +3 -0
  60. package/dist/schema.d.ts.map +1 -0
  61. package/dist/schema.js +39 -0
  62. package/dist/schema.js.map +1 -0
  63. package/dist/templates.d.ts +11 -0
  64. package/dist/templates.d.ts.map +1 -0
  65. package/dist/templates.js +46 -0
  66. package/dist/templates.js.map +1 -0
  67. package/dist/vectors.d.ts +23 -0
  68. package/dist/vectors.d.ts.map +1 -0
  69. package/dist/vectors.js +2 -0
  70. package/dist/vectors.js.map +1 -0
  71. package/package.json +55 -4
@@ -0,0 +1,76 @@
1
+ import { agnes } from 'ml-hclust';
2
+ // Group-average clustering of the reports with a lead in the window that ends
3
+ // on the day. Average linkage cannot join two groups with no pair at or above
4
+ // the threshold, so the clustering runs on each connected component of the
5
+ // shortlist graph whose edges pass the threshold. Each new cluster keeps the
6
+ // id of the old event that it overlaps most (docs/adr/0002).
7
+ export function recluster(day, table, shortlist, scores, { threshold, windowDays }) {
8
+ const window = table.withLeadSince(day - windowDays);
9
+ const inWindow = new Set(window);
10
+ const components = new UnionFind(window);
11
+ // The graph also holds vectors that a failed add left behind. They have no
12
+ // report in the window.
13
+ for (const [id, neighbours] of shortlist.ofDays(day - windowDays, day))
14
+ if (inWindow.has(id))
15
+ for (const neighbour of neighbours)
16
+ if (inWindow.has(neighbour.id) &&
17
+ scores.score(id, neighbour.id, neighbour.similarity) >= threshold)
18
+ components.union(id, neighbour.id);
19
+ const clusters = [];
20
+ for (const component of components.sets()) {
21
+ if (component.length === 1) {
22
+ clusters.push(component);
23
+ continue;
24
+ }
25
+ const distances = component.map((a) => component.map((b) => (a === b ? 0 : 1 - scores.score(a, b))));
26
+ for (const cluster of agnes(distances, {
27
+ method: 'average',
28
+ isDistanceMatrix: true,
29
+ }).cut(1 - threshold))
30
+ clusters.push(cluster.indices().map((index) => component[index] ?? ''));
31
+ }
32
+ carryIds(clusters, table);
33
+ }
34
+ // The largest overlaps choose first. Each old id goes to at most one
35
+ // cluster. Equal overlaps go to the lower old id, then to the cluster whose
36
+ // component has the lowest report id (docs/adr/0002). A cluster with no old
37
+ // id left gets a new id.
38
+ function carryIds(clusters, table) {
39
+ const overlaps = clusters.flatMap((cluster, index) => [...Map.groupBy(cluster, (id) => table.eventOf(id)).entries()].map(([event, members]) => ({ index, event, size: members.length })));
40
+ overlaps.sort((a, b) => b.size - a.size || a.event - b.event || a.index - b.index);
41
+ const idOf = new Map();
42
+ const used = new Set();
43
+ for (const { index, event } of overlaps)
44
+ if (!idOf.has(index) && !used.has(event)) {
45
+ idOf.set(index, event);
46
+ used.add(event);
47
+ }
48
+ for (const [index, cluster] of clusters.entries()) {
49
+ const event = idOf.get(index) ?? table.newEvent();
50
+ for (const id of cluster)
51
+ table.move(id, event);
52
+ }
53
+ }
54
+ class UnionFind {
55
+ #parent;
56
+ constructor(ids) {
57
+ this.#parent = new Map(ids.map((id) => [id, id]));
58
+ }
59
+ union(a, b) {
60
+ this.#parent.set(this.#find(a), this.#find(b));
61
+ }
62
+ // The sets, each in the order of the ids, in the order of their first id.
63
+ sets() {
64
+ return [
65
+ ...Map.groupBy(this.#parent.keys(), (id) => this.#find(id)).values(),
66
+ ];
67
+ }
68
+ #find(id) {
69
+ let root = id;
70
+ while (this.#parent.get(root) !== root)
71
+ root = this.#parent.get(root) ?? id;
72
+ this.#parent.set(id, root);
73
+ return root;
74
+ }
75
+ }
76
+ //# sourceMappingURL=recluster.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"recluster.js","sourceRoot":"","sources":["../../src/grouping/recluster.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAOlC,8EAA8E;AAC9E,8EAA8E;AAC9E,2EAA2E;AAC3E,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,UAAU,SAAS,CACvB,GAAW,EACX,KAAiB,EACjB,SAAoB,EACpB,MAAkB,EAClB,EAAE,SAAS,EAAE,UAAU,EAAU;IAEjC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC;IACzC,2EAA2E;IAC3E,wBAAwB;IACxB,KAAK,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,UAAU,EAAE,GAAG,CAAC;QACpE,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,KAAK,MAAM,SAAS,IAAI,UAAU;gBAChC,IACE,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,UAAU,CAAC,IAAI,SAAS;oBAEjE,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAE3C,MAAM,QAAQ,GAAe,EAAE,CAAC;IAChC,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACpC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,SAAS,EAAE;YACrC,MAAM,EAAE,SAAS;YACjB,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,qEAAqE;AACrE,4EAA4E;AAC5E,4EAA4E;AAC5E,yBAAyB;AACzB,SAAS,QAAQ,CAAC,QAA6B,EAAE,KAAiB;IAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CACnD,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAChE,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAC/D,CACF,CAAC;IACF,QAAQ,CAAC,IAAI,CACX,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CACpE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,QAAQ;QACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;IACH,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAClD,KAAK,MAAM,EAAE,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,MAAM,SAAS;IACJ,OAAO,CAAsB;IAEtC,YAAY,GAAsB;QAChC,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,CAAS,EAAE,CAAS;QACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,0EAA0E;IAC1E,IAAI;QACF,OAAO;YACL,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE;SACrE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,EAAU;QACd,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI;YAAE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
@@ -0,0 +1,11 @@
1
+ import type { Design } from '../design.ts';
2
+ import type { EventTable } from '../event-table.ts';
3
+ import { type Evidence } from '../pair-score.ts';
4
+ import type { Vectors } from '../vectors.ts';
5
+ export declare class PairScores {
6
+ #private;
7
+ constructor(table: EventTable, vectors: Vectors, model: Design['pairScore']);
8
+ remember(id: string, evidence: Evidence): void;
9
+ score(a: string, b: string, similarity?: number): number;
10
+ }
11
+ //# sourceMappingURL=scores.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scores.d.ts","sourceRoot":"","sources":["../../src/grouping/scores.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,KAAK,QAAQ,EAA2B,MAAM,kBAAkB,CAAC;AAC1E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAG7C,qBAAa,UAAU;;gBAOT,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC;IAO3E,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAM9C,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM;CAyBzD"}
@@ -0,0 +1,38 @@
1
+ import { pairFeatures, pairScore } from "../pair-score.js";
2
+ // The pair scores of one add, each one calculated once.
3
+ export class PairScores {
4
+ #table;
5
+ #vectors;
6
+ #model;
7
+ #evidence = new Map();
8
+ #scores = new Map();
9
+ constructor(table, vectors, model) {
10
+ this.#table = table;
11
+ this.#vectors = vectors;
12
+ this.#model = model;
13
+ }
14
+ // The evidence of a report that is not in the table yet.
15
+ remember(id, evidence) {
16
+ this.#evidence.set(id, evidence);
17
+ }
18
+ // similarity: the cosine similarity of a and b, when the caller has it
19
+ // already. Without it, the score asks the vectors.
20
+ score(a, b, similarity) {
21
+ const key = a < b ? JSON.stringify([a, b]) : JSON.stringify([b, a]);
22
+ let score = this.#scores.get(key);
23
+ if (score === undefined) {
24
+ score = pairScore(this.#model, pairFeatures(similarity ?? this.#vectors.similarity(a, b), this.#evidenceOf(a), this.#evidenceOf(b)));
25
+ this.#scores.set(key, score);
26
+ }
27
+ return score;
28
+ }
29
+ #evidenceOf(id) {
30
+ let evidence = this.#evidence.get(id);
31
+ if (evidence === undefined) {
32
+ evidence = this.#table.evidenceOf(id);
33
+ this.#evidence.set(id, evidence);
34
+ }
35
+ return evidence;
36
+ }
37
+ }
38
+ //# sourceMappingURL=scores.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scores.js","sourceRoot":"","sources":["../../src/grouping/scores.ts"],"names":[],"mappings":"AAEA,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAG1E,wDAAwD;AACxD,MAAM,OAAO,UAAU;IACZ,MAAM,CAAa;IACnB,QAAQ,CAAU;IAClB,MAAM,CAAsB;IAC5B,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;IACxC,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE7C,YAAY,KAAiB,EAAE,OAAgB,EAAE,KAA0B;QACzE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,yDAAyD;IACzD,QAAQ,CAAC,EAAU,EAAE,QAAkB;QACrC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,uEAAuE;IACvE,mDAAmD;IACnD,KAAK,CAAC,CAAS,EAAE,CAAS,EAAE,UAAmB;QAC7C,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,SAAS,CACf,IAAI,CAAC,MAAM,EACX,YAAY,CACV,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,EAC5C,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EACnB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CACpB,CACF,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACtC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
@@ -0,0 +1,10 @@
1
+ import type { Design } from '../design.ts';
2
+ import type { EventTable } from '../event-table.ts';
3
+ import type { Neighbour, Vectors } from '../vectors.ts';
4
+ export declare class Shortlist {
5
+ #private;
6
+ constructor(table: EventTable, vectors: Vectors, { shortlistSize, windowDays }: Design, waiting: number);
7
+ of(id: string): Neighbour[];
8
+ ofDays(first: number, last: number): Map<string, Neighbour[]>;
9
+ }
10
+ //# sourceMappingURL=shortlist.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shortlist.d.ts","sourceRoot":"","sources":["../../src/grouping/shortlist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAOxD,qBAAa,SAAS;;gBASlB,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,OAAO,EAChB,EAAE,aAAa,EAAE,UAAU,EAAE,EAAE,MAAM,EACrC,OAAO,EAAE,MAAM;IASjB,EAAE,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,EAAE;IAS3B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;CAiC9D"}
@@ -0,0 +1,58 @@
1
+ import { DAY_MS } from "../day.js";
2
+ // The nearest reports of a report, from the reports that have an event. An
3
+ // add upserts all its reports before it gives them events, so the vectors
4
+ // also hold reports with no event yet. The shortlist asks for more neighbours
5
+ // until it has shortlistSize reports with an event, or the window has no
6
+ // more reports.
7
+ export class Shortlist {
8
+ #table;
9
+ #vectors;
10
+ #size;
11
+ #windowDays;
12
+ #k;
13
+ // waiting: the number of reports of the add that have no event yet.
14
+ constructor(table, vectors, { shortlistSize, windowDays }, waiting) {
15
+ this.#table = table;
16
+ this.#vectors = vectors;
17
+ this.#size = shortlistSize;
18
+ this.#windowDays = windowDays;
19
+ this.#k = shortlistSize + waiting;
20
+ }
21
+ of(id) {
22
+ return this.#widen(id, this.#vectors.nearest(id, { k: this.#k, withinDays: this.#windowDays }));
23
+ }
24
+ // The shortlists of the reports published from the first day to the last
25
+ // day, from one neighbour graph of the vectors.
26
+ ofDays(first, last) {
27
+ const graph = this.#vectors.neighbourGraph({
28
+ from: new Date(first * DAY_MS),
29
+ to: new Date(last * DAY_MS),
30
+ k: this.#k,
31
+ withinDays: this.#windowDays,
32
+ });
33
+ return new Map(Iterator.from(graph).map(([id, neighbours]) => [
34
+ id,
35
+ this.#widen(id, neighbours),
36
+ ]));
37
+ }
38
+ // first: the neighbours for this.#k. Each next ask doubles k.
39
+ #widen(id, first) {
40
+ let neighbours = first;
41
+ let k = this.#k;
42
+ while (true) {
43
+ const kept = neighbours
44
+ .values()
45
+ .filter((neighbour) => this.#table.has(neighbour.id))
46
+ .take(this.#size)
47
+ .toArray();
48
+ if (kept.length === this.#size || neighbours.length < k)
49
+ return kept;
50
+ k *= 2;
51
+ neighbours = this.#vectors.nearest(id, {
52
+ k,
53
+ withinDays: this.#windowDays,
54
+ });
55
+ }
56
+ }
57
+ }
58
+ //# sourceMappingURL=shortlist.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shortlist.js","sourceRoot":"","sources":["../../src/grouping/shortlist.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAKnC,2EAA2E;AAC3E,0EAA0E;AAC1E,8EAA8E;AAC9E,yEAAyE;AACzE,gBAAgB;AAChB,MAAM,OAAO,SAAS;IACX,MAAM,CAAa;IACnB,QAAQ,CAAU;IAClB,KAAK,CAAS;IACd,WAAW,CAAS;IACpB,EAAE,CAAS;IAEpB,oEAAoE;IACpE,YACE,KAAiB,EACjB,OAAgB,EAChB,EAAE,aAAa,EAAE,UAAU,EAAU,EACrC,OAAe;QAEf,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,aAAa,GAAG,OAAO,CAAC;IACpC,CAAC;IAED,EAAE,CAAC,EAAU;QACX,OAAO,IAAI,CAAC,MAAM,CAChB,EAAE,EACF,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CACxE,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,gDAAgD;IAChD,MAAM,CAAC,KAAa,EAAE,IAAY;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;YACzC,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YAC9B,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;YAC3B,CAAC,EAAE,IAAI,CAAC,EAAE;YACV,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC,CAAC;QACH,OAAO,IAAI,GAAG,CACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;YAC7C,EAAE;YACF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC;SAC5B,CAAC,CACH,CAAC;IACJ,CAAC;IAED,8DAA8D;IAC9D,MAAM,CAAC,EAAU,EAAE,KAA2B;QAC5C,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QAChB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,UAAU;iBACpB,MAAM,EAAE;iBACR,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;iBACpD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;iBAChB,OAAO,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YACrE,CAAC,IAAI,CAAC,CAAC;YACP,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE;gBACrC,CAAC;gBACD,UAAU,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,5 @@
1
+ export { type Design } from './design.ts';
2
+ export { design3152b } from './design-3152b.ts';
3
+ export { type Event, EventDetector, type EventDetectorOptions, type Report, } from './event-detector.ts';
4
+ export type { Vectors } from './vectors.ts';
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,KAAK,KAAK,EACV,aAAa,EACb,KAAK,oBAAoB,EACzB,KAAK,MAAM,GACZ,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,4 @@
1
+ export {} from "./design.js";
2
+ export { design3152b } from "./design-3152b.js";
3
+ export { EventDetector, } from "./event-detector.js";
4
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,aAAa,GAGd,MAAM,qBAAqB,CAAC"}
package/dist/lead.d.ts ADDED
@@ -0,0 +1,15 @@
1
+ import type { Design } from './design.ts';
2
+ type Rule = Design['lead'];
3
+ export type Lead = {
4
+ readonly title: string | null;
5
+ readonly sentences: readonly string[];
6
+ };
7
+ export type IsTemplate = (key: string) => boolean;
8
+ export declare function leadOf(title: string, body: string, rule: Rule, isTemplate?: IsTemplate): Lead;
9
+ export declare function segmentsOf(body: string, rule: Rule): {
10
+ readonly segment: string;
11
+ readonly key: string;
12
+ }[];
13
+ export declare function numberText(title: string, body: string, rule: Rule, isTemplate?: IsTemplate): string;
14
+ export {};
15
+ //# sourceMappingURL=lead.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lead.d.ts","sourceRoot":"","sources":["../src/lead.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,KAAK,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AAI3B,MAAM,MAAM,IAAI,GAAG;IAEjB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC,CAAC;AAIF,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;AAMlD,wBAAgB,MAAM,CACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,UAAU,GAAE,UAAiB,GAC5B,IAAI,CAIN;AAKD,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,GACT;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,CAMtD;AAID,wBAAgB,UAAU,CACxB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,IAAI,EACV,UAAU,GAAE,UAAiB,GAC5B,MAAM,CAMR"}
package/dist/lead.js ADDED
@@ -0,0 +1,74 @@
1
+ const segmenter = new Intl.Segmenter('en', { granularity: 'sentence' });
2
+ const none = () => false;
3
+ // A report without a title takes its first real sentence as the title.
4
+ export function leadOf(title, body, rule, isTemplate = none) {
5
+ const real = realSentences(body, title, rule, isTemplate);
6
+ const resolved = title ? clean(title) : (real.shift() ?? null);
7
+ return { title: resolved, sentences: real.slice(0, rule.sentences) };
8
+ }
9
+ // The segments of a body, in order, each with its key: the segment in lower
10
+ // case, with one space between words. With a clean rule, the URLs go away
11
+ // before the sentence split, so a "?" or a "." in a URL splits no sentence.
12
+ export function segmentsOf(body, rule) {
13
+ const text = rule.clean === undefined ? body : body.replace(urls(rule), ' ');
14
+ return [...segmenter.segment(text)].flatMap(({ segment }) => {
15
+ const key = clean(segment).toLowerCase();
16
+ return key ? [{ segment, key }] : [];
17
+ });
18
+ }
19
+ // The text that the numbers of a report come from: the title and the body.
20
+ // With a clean rule, the URLs and the template segments are not in it.
21
+ export function numberText(title, body, rule, isTemplate = none) {
22
+ if (rule.clean === undefined)
23
+ return `${title}\n${body}`;
24
+ const kept = segmentsOf(body, rule)
25
+ .filter(({ key }) => !isTemplate(key))
26
+ .map(({ segment }) => segment);
27
+ return `${title}\n${kept.join(' ')}`;
28
+ }
29
+ // The real sentences of a body, in order, without their URLs and without the
30
+ // template segments of its publisher. A sentence that starts with the title
31
+ // loses that prefix.
32
+ function realSentences(body, title, rule, isTemplate) {
33
+ const prefix = title ? clean(title).toLowerCase() : null;
34
+ const sentences = [];
35
+ for (const { segment, key } of segmentsOf(body, rule)) {
36
+ if (isTemplate(key))
37
+ continue;
38
+ let sentence = clean(segment);
39
+ if (!isReal(sentence, rule))
40
+ continue;
41
+ sentence = clean(sentence.replace(urls(rule), ' '));
42
+ if (prefix !== null && sentence.toLowerCase().startsWith(prefix))
43
+ sentence = clean(sentence.slice(prefix.length));
44
+ if (sentence && isReal(sentence, rule))
45
+ sentences.push(sentence);
46
+ }
47
+ return sentences;
48
+ }
49
+ // A segment is a real sentence when URLs are at most half of its characters,
50
+ // and the rest has enough words and is mostly letters. Blank lines, captions,
51
+ // bylines and URL lines are not real sentences.
52
+ function isReal(segment, rule) {
53
+ const compact = segment.replace(/\s+/gu, '');
54
+ if (!compact)
55
+ return false;
56
+ const urlCharacters = (compact.match(urls(rule)) ?? []).reduce((sum, url) => sum + url.length, 0);
57
+ if (urlCharacters / compact.length > rule.maxUrlShare)
58
+ return false;
59
+ const rest = segment.replace(urls(rule), ' ');
60
+ if ((rest.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length < rule.minWords)
61
+ return false;
62
+ const characters = rest.replace(/\s/gu, '');
63
+ const letters = (rest.match(/\p{L}/gu) ?? []).length;
64
+ return letters / characters.length >= rule.minLetterRatio;
65
+ }
66
+ function clean(text) {
67
+ return text.replace(/\s+/gu, ' ').trim();
68
+ }
69
+ // A new expression for each use, because a global expression keeps its
70
+ // lastIndex.
71
+ function urls(rule) {
72
+ return new RegExp(rule.urlPattern, 'giu');
73
+ }
74
+ //# sourceMappingURL=lead.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lead.js","sourceRoot":"","sources":["../src/lead.ts"],"names":[],"mappings":"AAgBA,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;AACxE,MAAM,IAAI,GAAe,GAAG,EAAE,CAAC,KAAK,CAAC;AAErC,uEAAuE;AACvE,MAAM,UAAU,MAAM,CACpB,KAAa,EACb,IAAY,EACZ,IAAU,EACV,aAAyB,IAAI;IAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC;IAC/D,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;AACvE,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,4EAA4E;AAC5E,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,IAAU;IAEV,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;QAC1D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QACzC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,2EAA2E;AAC3E,uEAAuE;AACvE,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,IAAY,EACZ,IAAU,EACV,aAAyB,IAAI;IAE7B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;SAChC,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;IACjC,OAAO,GAAG,KAAK,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACvC,CAAC;AAED,6EAA6E;AAC7E,4EAA4E;AAC5E,qBAAqB;AACrB,SAAS,aAAa,CACpB,IAAY,EACZ,KAAa,EACb,IAAU,EACV,UAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACtD,IAAI,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC9B,IAAI,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC;YAAE,SAAS;QACtC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;YAC9D,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAClD,IAAI,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,6EAA6E;AAC7E,8EAA8E;AAC9E,gDAAgD;AAChD,SAAS,MAAM,CAAC,OAAe,EAAE,IAAU;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5D,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,EAC9B,CAAC,CACF,CAAC;IACF,IAAI,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ;QACtE,OAAO,KAAK,CAAC;IACf,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACrD,OAAO,OAAO,GAAG,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC;AAC5D,CAAC;AAED,SAAS,KAAK,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,uEAAuE;AACvE,aAAa;AACb,SAAS,IAAI,CAAC,IAAU;IACtB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function numbersOf(text: string): Set<string>;
2
+ //# sourceMappingURL=numbers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"numbers.d.ts","sourceRoot":"","sources":["../src/numbers.ts"],"names":[],"mappings":"AAGA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAYnD"}
@@ -0,0 +1,19 @@
1
+ // The numbers and version-like tokens of a text, such as "4.5", "rc1", "v14",
2
+ // "143,000" and "2017". A thousands separator and a leading "v" go away, so
3
+ // "143,000" is "143000" and "v14" is "14".
4
+ export function numbersOf(text) {
5
+ const numbers = new Set();
6
+ for (const [match] of text
7
+ .toLowerCase()
8
+ .matchAll(/[a-z]*\d[\d.,]*[a-z\d]*/gu)) {
9
+ let token = match
10
+ .replace(/[.,]+$/u, '')
11
+ .replace(/(\d),(?=\d{3}\b)/gu, '$1');
12
+ if (/^v\d/u.test(token))
13
+ token = token.slice(1);
14
+ if (token)
15
+ numbers.add(token);
16
+ }
17
+ return numbers;
18
+ }
19
+ //# sourceMappingURL=numbers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"numbers.js","sourceRoot":"","sources":["../src/numbers.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,2CAA2C;AAC3C,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI;SACvB,WAAW,EAAE;SACb,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,KAAK;aACd,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;aACtB,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
@@ -0,0 +1,11 @@
1
+ import type { Design } from './design.ts';
2
+ export type Evidence = {
3
+ readonly numbers: ReadonlySet<string>;
4
+ readonly publisher: string | null;
5
+ };
6
+ export declare const PAIR_FEATURES: readonly ["similarity", "agreeingNumbers", "disagreeingNumbers", "samePublisher", "samePublisherSimilarity"];
7
+ export type PairFeature = (typeof PAIR_FEATURES)[number];
8
+ export type PairFeatures = Record<PairFeature, number>;
9
+ export declare function pairFeatures(similarity: number, a: Evidence, b: Evidence): PairFeatures;
10
+ export declare function pairScore(model: Design['pairScore'], features: PairFeatures): number;
11
+ //# sourceMappingURL=pair-score.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pair-score.d.ts","sourceRoot":"","sources":["../src/pair-score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,MAAM,MAAM,QAAQ,GAAG;IAErB,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAEtC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAGF,eAAO,MAAM,aAAa,8GAMhB,CAAC;AAEX,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;AAKvD,wBAAgB,YAAY,CAC1B,UAAU,EAAE,MAAM,EAClB,CAAC,EAAE,QAAQ,EACX,CAAC,EAAE,QAAQ,GACV,YAAY,CAed;AASD,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,EAC1B,QAAQ,EAAE,YAAY,GACrB,MAAM,CAQR"}
@@ -0,0 +1,39 @@
1
+ // The features of the pair score, in the order of its terms.
2
+ export const PAIR_FEATURES = [
3
+ 'similarity',
4
+ 'agreeingNumbers',
5
+ 'disagreeingNumbers',
6
+ 'samePublisher',
7
+ 'samePublisherSimilarity',
8
+ ];
9
+ // The features of two reports, from the cosine similarity of their leads, the
10
+ // numbers of their full texts and their publishers. The detector scores a
11
+ // pair with them, and the fitter (eval/fit) fits the weights on them.
12
+ export function pairFeatures(similarity, a, b) {
13
+ const shared = [...a.numbers].filter((number) => b.numbers.has(number)).length;
14
+ const samePublisher = a.publisher !== null && a.publisher === b.publisher ? 1 : 0;
15
+ return {
16
+ similarity,
17
+ agreeingNumbers: Math.log1p(shared),
18
+ disagreeingNumbers: Math.log1p(Math.min(a.numbers.size - shared, b.numbers.size - shared)),
19
+ samePublisher,
20
+ samePublisherSimilarity: samePublisher * similarity,
21
+ };
22
+ }
23
+ // The probability that two reports report one event. Numbers that both
24
+ // reports give are evidence for one event. Numbers that only one report gives
25
+ // are evidence against it (the Fellegi and Sunter model). One publisher is
26
+ // evidence against one event: at the same similarity, two reports of one
27
+ // publisher report one event less often than two reports of two publishers,
28
+ // because a publisher writes many reports on one topic (docs/adr/0005). A
29
+ // term that the model does not have is not in the score.
30
+ export function pairScore(model, features) {
31
+ const z = PAIR_FEATURES.reduce((sum, feature) => {
32
+ const term = model[feature];
33
+ return term === undefined
34
+ ? sum
35
+ : sum + (term.weight * (features[feature] - term.mean)) / term.sd;
36
+ }, model.intercept);
37
+ return 1 / (1 + Math.exp(-z));
38
+ }
39
+ //# sourceMappingURL=pair-score.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pair-score.js","sourceRoot":"","sources":["../src/pair-score.ts"],"names":[],"mappings":"AAUA,6DAA6D;AAC7D,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY;IACZ,iBAAiB;IACjB,oBAAoB;IACpB,eAAe;IACf,yBAAyB;CACjB,CAAC;AAMX,8EAA8E;AAC9E,0EAA0E;AAC1E,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAC1B,UAAkB,EAClB,CAAW,EACX,CAAW;IAEX,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC9C,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CACtB,CAAC,MAAM,CAAC;IACT,MAAM,aAAa,GACjB,CAAC,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,OAAO;QACL,UAAU;QACV,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACnC,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,CAC3D;QACD,aAAa;QACb,uBAAuB,EAAE,aAAa,GAAG,UAAU;KACpD,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,8EAA8E;AAC9E,2EAA2E;AAC3E,yEAAyE;AACzE,4EAA4E;AAC5E,0EAA0E;AAC1E,yDAAyD;AACzD,MAAM,UAAU,SAAS,CACvB,KAA0B,EAC1B,QAAsB;IAEtB,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,IAAI,KAAK,SAAS;YACvB,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;IACtE,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Declaration } from '@zukhruf/sqlite-drift';
2
+ export declare const eventSchema: Declaration;
3
+ //# sourceMappingURL=schema.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAMpD,eAAO,MAAM,WAAW,aAmCvB,CAAC"}
package/dist/schema.js ADDED
@@ -0,0 +1,39 @@
1
+ import { Declaration } from '@zukhruf/sqlite-drift';
2
+ // The event of each report, the clock of the detector, and the segments of
3
+ // the reports of each publisher (src/templates.ts). The rows derive
4
+ // from the reports, so a stale table is replaced with its rows: the caller
5
+ // adds the reports again.
6
+ export const eventSchema = new Declaration([
7
+ `CREATE TABLE event_report (
8
+ id TEXT PRIMARY KEY,
9
+ event INTEGER NOT NULL,
10
+ -- Milliseconds since the Unix epoch.
11
+ published_at INTEGER NOT NULL,
12
+ -- The UTC publish day: whole days since the Unix epoch.
13
+ day INTEGER NOT NULL,
14
+ -- 1 when the report has a lead, 0 for a headline-only report.
15
+ has_lead INTEGER NOT NULL,
16
+ -- The numbers of the title and the body, with a space between them. A
17
+ -- number has no space in it.
18
+ numbers TEXT NOT NULL,
19
+ -- Null when the caller gave no publisher.
20
+ publisher TEXT
21
+ ) STRICT`,
22
+ `CREATE INDEX event_report_event ON event_report (event)`,
23
+ `CREATE INDEX event_report_day ON event_report (day)`,
24
+ `CREATE TABLE event_clock (
25
+ id INTEGER PRIMARY KEY CHECK (id = 1),
26
+ -- The latest publish day that the detector got.
27
+ current_day INTEGER NOT NULL,
28
+ -- The id of the next new event. An event id is never used again.
29
+ next_event INTEGER NOT NULL
30
+ ) STRICT`,
31
+ `CREATE TABLE event_segment (
32
+ publisher TEXT NOT NULL,
33
+ -- A hash of the segment in lower case, with one space between words.
34
+ segment TEXT NOT NULL,
35
+ report TEXT NOT NULL,
36
+ PRIMARY KEY (publisher, segment, report)
37
+ ) STRICT, WITHOUT ROWID`,
38
+ ], { rows: 'replace' });
39
+ //# sourceMappingURL=schema.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,2EAA2E;AAC3E,oEAAoE;AACpE,2EAA2E;AAC3E,0BAA0B;AAC1B,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,WAAW,CACxC;IACE;;;;;;;;;;;;;;SAcK;IACL,yDAAyD;IACzD,qDAAqD;IACrD;;;;;;SAMK;IACL;;;;;;wBAMoB;CACrB,EACD,EAAE,IAAI,EAAE,SAAS,EAAE,CACpB,CAAC"}
@@ -0,0 +1,11 @@
1
+ import type { DatabaseSync } from 'node:sqlite';
2
+ import type { Design } from './design.ts';
3
+ import type { Report } from './event-detector.ts';
4
+ import { type IsTemplate } from './lead.ts';
5
+ export declare class Templates {
6
+ #private;
7
+ constructor(database: DatabaseSync, rule: Design['lead']);
8
+ observe(reports: readonly Report[]): void;
9
+ of(publisher: string | undefined): IsTemplate;
10
+ }
11
+ //# sourceMappingURL=templates.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"templates.d.ts","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,KAAK,UAAU,EAAc,MAAM,WAAW,CAAC;AAWxD,qBAAa,SAAS;;gBAIR,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC;IAQxD,OAAO,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAezC,EAAE,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU;CAa9C"}
@@ -0,0 +1,46 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { segmentsOf } from "./lead.js";
3
+ const none = () => false;
4
+ // The segments that a publisher repeats in its reports: banners, promotions,
5
+ // player widgets and other text of the site template. A segment is a
6
+ // template when at least templateReports reports of one publisher hold it.
7
+ // This is site-level template detection (Bar-Yossef and Rajagopalan, WWW
8
+ // 2002): text that many pages of one site share is not the content of a
9
+ // page. The detector keeps the key of each segment of each report with a
10
+ // publisher in event_segment.
11
+ export class Templates {
12
+ #database;
13
+ #rule;
14
+ constructor(database, rule) {
15
+ this.#database = database;
16
+ this.#rule = rule;
17
+ }
18
+ // Keeps the segments of each report with a publisher. Call it for all the
19
+ // reports of an add before their leads, so the reports of one add count
20
+ // for each other.
21
+ observe(reports) {
22
+ if (this.#rule.clean?.templateReports == null)
23
+ return;
24
+ const insert = this.#database.prepare('INSERT OR IGNORE INTO "event_segment" ("publisher", "segment", "report") VALUES (?, ?, ?)');
25
+ for (const report of reports)
26
+ if (report.publisher !== undefined)
27
+ for (const key of new Set(segmentsOf(report.body, this.#rule).map(({ key }) => key)))
28
+ insert.run(report.publisher, hash(key), report.id);
29
+ }
30
+ // The test of the template segments of one publisher. A report without a
31
+ // publisher has no template segment.
32
+ of(publisher) {
33
+ const reports = this.#rule.clean?.templateReports;
34
+ if (reports == null || publisher === undefined)
35
+ return none;
36
+ const templates = new Set(this.#database
37
+ .prepare('SELECT "segment" FROM "event_segment" WHERE "publisher" = ? GROUP BY "segment" HAVING count(*) >= ?')
38
+ .all(publisher, reports)
39
+ .map((row) => String(row.segment)));
40
+ return templates.size === 0 ? none : (key) => templates.has(hash(key));
41
+ }
42
+ }
43
+ function hash(key) {
44
+ return createHash('sha256').update(key).digest('base64url').slice(0, 22);
45
+ }
46
+ //# sourceMappingURL=templates.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"templates.js","sourceRoot":"","sources":["../src/templates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAKzC,OAAO,EAAmB,UAAU,EAAE,MAAM,WAAW,CAAC;AAExD,MAAM,IAAI,GAAe,GAAG,EAAE,CAAC,KAAK,CAAC;AAErC,6EAA6E;AAC7E,qEAAqE;AACrE,2EAA2E;AAC3E,yEAAyE;AACzE,wEAAwE;AACxE,yEAAyE;AACzE,8BAA8B;AAC9B,MAAM,OAAO,SAAS;IACX,SAAS,CAAe;IACxB,KAAK,CAAiB;IAE/B,YAAY,QAAsB,EAAE,IAAoB;QACtD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,0EAA0E;IAC1E,wEAAwE;IACxE,kBAAkB;IAClB,OAAO,CAAC,OAA0B;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,eAAe,IAAI,IAAI;YAAE,OAAO;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CACnC,2FAA2F,CAC5F,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO;YAC1B,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;gBAChC,KAAK,MAAM,GAAG,IAAI,IAAI,GAAG,CACvB,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAC1D;oBACC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,yEAAyE;IACzE,qCAAqC;IACrC,EAAE,CAAC,SAA6B;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,eAAe,CAAC;QAClD,IAAI,OAAO,IAAI,IAAI,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,IAAI,CAAC,SAAS;aACX,OAAO,CACN,qGAAqG,CACtG;aACA,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC;aACvB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CACrC,CAAC;QACF,OAAO,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACzE,CAAC;CACF;AAED,SAAS,IAAI,CAAC,GAAW;IACvB,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3E,CAAC"}
@@ -0,0 +1,23 @@
1
+ export type Neighbour = {
2
+ readonly id: string;
3
+ readonly similarity: number;
4
+ };
5
+ export interface Vectors {
6
+ upsert(documents: readonly {
7
+ readonly id: string;
8
+ readonly text: string;
9
+ readonly publishedAt: Date;
10
+ }[]): Promise<void>;
11
+ nearest(id: string, options: {
12
+ readonly k: number;
13
+ readonly withinDays: number;
14
+ }): readonly Neighbour[];
15
+ neighbourGraph(options: {
16
+ readonly from: Date;
17
+ readonly to: Date;
18
+ readonly k: number;
19
+ readonly withinDays: number;
20
+ }): Iterable<readonly [string, readonly Neighbour[]]>;
21
+ similarity(a: string, b: string): number;
22
+ }
23
+ //# sourceMappingURL=vectors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vectors.d.ts","sourceRoot":"","sources":["../src/vectors.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAK7E,MAAM,WAAW,OAAO;IACtB,MAAM,CACJ,SAAS,EAAE,SAAS;QAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;KAC5B,EAAE,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAGjB,OAAO,CACL,EAAE,EAAE,MAAM,EACV,OAAO,EAAE;QAAE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAC3D,SAAS,SAAS,EAAE,CAAC;IAGxB,cAAc,CAAC,OAAO,EAAE;QACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;KAC7B,GAAG,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC,CAAC,CAAC;IAEtD,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1C"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=vectors.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vectors.js","sourceRoot":"","sources":["../src/vectors.ts"],"names":[],"mappings":""}