@zukhruf/event-detection 0.0.0-stage → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +110 -2
- package/dist/analyse.d.ts +6 -0
- package/dist/analyse.d.ts.map +1 -0
- package/dist/analyse.js +26 -0
- package/dist/analyse.js.map +1 -0
- package/dist/day.d.ts +2 -0
- package/dist/day.d.ts.map +1 -0
- package/dist/day.js +3 -0
- package/dist/day.js.map +1 -0
- package/dist/design-3152b.d.ts +3 -0
- package/dist/design-3152b.d.ts.map +1 -0
- package/dist/design-3152b.js +47 -0
- package/dist/design-3152b.js.map +1 -0
- package/dist/design.d.ts +33 -0
- package/dist/design.d.ts.map +1 -0
- package/dist/design.js +2 -0
- package/dist/design.js.map +1 -0
- package/dist/event-detector.d.ts +30 -0
- package/dist/event-detector.d.ts.map +1 -0
- package/dist/event-detector.js +69 -0
- package/dist/event-detector.js.map +1 -0
- package/dist/event-table.d.ts +28 -0
- package/dist/event-table.d.ts.map +1 -0
- package/dist/event-table.js +89 -0
- package/dist/event-table.js.map +1 -0
- package/dist/grouping/assign.d.ts +5 -0
- package/dist/grouping/assign.d.ts.map +1 -0
- package/dist/grouping/assign.js +25 -0
- package/dist/grouping/assign.js.map +1 -0
- package/dist/grouping/recluster.d.ts +6 -0
- package/dist/grouping/recluster.d.ts.map +1 -0
- package/dist/grouping/recluster.js +76 -0
- package/dist/grouping/recluster.js.map +1 -0
- package/dist/grouping/scores.d.ts +11 -0
- package/dist/grouping/scores.d.ts.map +1 -0
- package/dist/grouping/scores.js +38 -0
- package/dist/grouping/scores.js.map +1 -0
- package/dist/grouping/shortlist.d.ts +10 -0
- package/dist/grouping/shortlist.d.ts.map +1 -0
- package/dist/grouping/shortlist.js +58 -0
- package/dist/grouping/shortlist.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/lead.d.ts +15 -0
- package/dist/lead.d.ts.map +1 -0
- package/dist/lead.js +74 -0
- package/dist/lead.js.map +1 -0
- package/dist/numbers.d.ts +2 -0
- package/dist/numbers.d.ts.map +1 -0
- package/dist/numbers.js +19 -0
- package/dist/numbers.js.map +1 -0
- package/dist/pair-score.d.ts +11 -0
- package/dist/pair-score.d.ts.map +1 -0
- package/dist/pair-score.js +39 -0
- package/dist/pair-score.js.map +1 -0
- package/dist/schema.d.ts +3 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +39 -0
- package/dist/schema.js.map +1 -0
- package/dist/templates.d.ts +11 -0
- package/dist/templates.d.ts.map +1 -0
- package/dist/templates.js +46 -0
- package/dist/templates.js.map +1 -0
- package/dist/vectors.d.ts +23 -0
- package/dist/vectors.d.ts.map +1 -0
- package/dist/vectors.js +2 -0
- package/dist/vectors.js.map +1 -0
- package/package.json +55 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ezzabuzaid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,111 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @zukhruf/event-detection
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Many sources report one real occurrence, and a reader wants to see it one time. This package puts the reports of one real occurrence into one event. It finds the nearest reports of each report by the vectors of their leads. It scores each pair with a frozen logistic model on the similarity, the numbers and the publishers of the two reports. On each new publish day, it clusters the reports of the last 7 days again with group average. This kind of program is also known as event detection, or event coreference across documents.
|
|
4
|
+
|
|
5
|
+
The words in these documents have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md).
|
|
6
|
+
|
|
7
|
+
## Use it
|
|
8
|
+
|
|
9
|
+
This project is an experiment. It uses `node:sqlite`. The package does not start a model, a server or a process. Give it a vectors port that is ready, for example a `Collection` of [`@zukhruf/embeddings`](../embeddings).
|
|
10
|
+
|
|
11
|
+
The parameters come from research rounds #3130 and #3152, and they are valid only with EmbeddingGemma 2 ([ADR 0001](./docs/adr/0001-the-parameters-are-frozen-by-research-round-3130.md), [ADR 0005](./docs/adr/0005-research-round-3152-adds-a-same-publisher-term-and-fits-the-weights-again.md)). Give the publisher of each report when you know it.
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
npm install @zukhruf/event-detection @zukhruf/embeddings ai
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
19
|
+
|
|
20
|
+
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
|
|
21
|
+
|
|
22
|
+
import { Collection } from '@zukhruf/embeddings';
|
|
23
|
+
import { EventDetector, design3152b } from '@zukhruf/event-detection';
|
|
24
|
+
|
|
25
|
+
const local = createOpenAICompatible({
|
|
26
|
+
name: 'llama.cpp',
|
|
27
|
+
baseURL: 'http://127.0.0.1:8080/v1',
|
|
28
|
+
});
|
|
29
|
+
using database = new DatabaseSync('events.sqlite');
|
|
30
|
+
const detector = new EventDetector({
|
|
31
|
+
database,
|
|
32
|
+
vectors: new Collection({
|
|
33
|
+
database,
|
|
34
|
+
model: local.embeddingModel(design3152b.embeddingModel),
|
|
35
|
+
}),
|
|
36
|
+
design: design3152b,
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
await detector.add([
|
|
40
|
+
{
|
|
41
|
+
id: 'a',
|
|
42
|
+
title: 'Storm closes the port',
|
|
43
|
+
body: 'A storm closed the port on Monday morning. Ships waited outside the harbour.',
|
|
44
|
+
publishedAt: new Date('2026-10-20T08:00:00Z'),
|
|
45
|
+
publisher: 'news.example',
|
|
46
|
+
},
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
detector.events({ since: new Date('2026-10-14') });
|
|
50
|
+
// [{ id: 0, members: ['a'], firstSeen: …, lastSeen: … }]
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## API
|
|
54
|
+
|
|
55
|
+
- `new EventDetector({ database, vectors, design })`. The caller opens and closes `database`. The detector makes its tables `event_report` and `event_clock` in it. `vectors` is the port to the vectors of this detector only. `design` is the recipe of one research round. The vectors must come from the embedding model that `design.embeddingModel` names.
|
|
56
|
+
- `design3152b` is the design of research round #3152b ([ADR 0005](./docs/adr/0005-research-round-3152-adds-a-same-publisher-term-and-fits-the-weights-again.md)). `Design` is its type.
|
|
57
|
+
- `add(reports)` gives each new report an event. A report has an `id`, a `title`, a `body`, a `publishedAt` date and, when the caller knows it, a `publisher`. The detector gives the lead of each report that has one to `vectors.upsert`, as `{ id, text, publishedAt }`. Then it gives the reports their events, in the order of their publish dates. A report whose id the detector has already is skipped.
|
|
58
|
+
- `events({ since })` gives the events with a report published at or after `since`, in the order of their first report. An event has an `id` (a number), its `members` (report ids, in the order of their publish dates), `firstSeen` and `lastSeen`.
|
|
59
|
+
- `Vectors` is the type of the port: `upsert(documents)`, `nearest(id, { k, withinDays })`, `neighbourGraph({ from, to, k, withinDays })` and `similarity(a, b)`. A clustering asks for one neighbour graph of the window. The `Collection` of `@zukhruf/embeddings` has this shape, with no adapter.
|
|
60
|
+
|
|
61
|
+
## How it decides
|
|
62
|
+
|
|
63
|
+
1. **Lead.** The detector takes the title and the first two real sentences of the body. A URL line, a byline or a caption is not a real sentence. A URL in a real sentence goes away.
|
|
64
|
+
2. **Shortlist.** The 80 nearest reports, by similarity, from the reports of 7 publish days before or after.
|
|
65
|
+
3. **Pair score.** A logistic regression on the similarity, the numbers that both reports give, the numbers that only one report gives, and the same publisher. The numbers come from the title and the whole body. Two reports of one publisher get a lower pair score. A report without a publisher is never of the same publisher as another report.
|
|
66
|
+
4. **Online assignment.** A new report joins the event of its shortlist with the highest mean pair score, when it is at least 0.5. Otherwise it starts a new event.
|
|
67
|
+
5. **Clustering.** When the first report of a new publish day arrives, the detector clusters the window of the last publish day again, with group average. Each new cluster keeps the event id of the old event that it overlaps most ([ADR 0002](./docs/adr/0002-the-window-is-clustered-again-on-each-new-publish-day.md)).
|
|
68
|
+
6. **Headline-only reports.** A report with no real sentence is alone in its event, always ([ADR 0003](./docs/adr/0003-a-headline-only-report-never-joins-another-report.md)).
|
|
69
|
+
|
|
70
|
+
## Result
|
|
71
|
+
|
|
72
|
+
On the Gun Violence Corpus (509 reports, 241 incidents), which no round had seen before, round #3130 measured:
|
|
73
|
+
|
|
74
|
+
| Score | Detector | Baseline |
|
|
75
|
+
| --------------------------------- | --------------------- | --------------------- |
|
|
76
|
+
| Pair precision / recall / F0.5 | 0.918 / 0.712 / 0.868 | 0.820 / 0.144 / 0.423 |
|
|
77
|
+
| B-cubed precision / recall / F0.5 | 0.977 / 0.849 / 0.948 | 0.978 / 0.554 / 0.848 |
|
|
78
|
+
|
|
79
|
+
Round #3152 added the same-publisher term and fitted the weights again on the training sets (ADR 0005). The package gives these scores of the round. The anthropic-outlets scores are in-sample, because the round fitted on that set:
|
|
80
|
+
|
|
81
|
+
| Set | Pair P / R / F0.5 | B-cubed P / R / F0.5 |
|
|
82
|
+
| --------------------------------------- | --------------------- | --------------------- |
|
|
83
|
+
| Gun Violence Corpus, design 3152 | 0.961 / 0.712 / 0.898 | 0.989 / 0.848 / 0.957 |
|
|
84
|
+
| anthropic-outlets, design 3130 | 0.306 / 0.313 / 0.307 | 0.800 / 0.865 / 0.812 |
|
|
85
|
+
| anthropic-outlets, design 3152 (fitted) | 0.641 / 0.257 / 0.493 | 0.938 / 0.853 / 0.919 |
|
|
86
|
+
|
|
87
|
+
On the exam set of round #3152 (deepmind-outlets, never seen before), the package gives CoNLL F0.5 0.883 against 0.818 for design 3130, and it passed the rule of the round ([ADR 0005](./docs/adr/0005-research-round-3152-adds-a-same-publisher-term-and-fits-the-weights-again.md)). A replay of the Gun Violence Corpus through the package gives the events of round #3152 ([src/conformance.test.ts](./src/conformance.test.ts)).
|
|
88
|
+
|
|
89
|
+
## Limits
|
|
90
|
+
|
|
91
|
+
Read [the limitations](./docs/limitations.md) before you use the events for a decision. In short:
|
|
92
|
+
|
|
93
|
+
- The events of the latest publish day are provisional until a report of a later publish day arrives.
|
|
94
|
+
- A report more than 7 publish days after its event starts a new event.
|
|
95
|
+
- Look-alike occurrences a few days apart can join.
|
|
96
|
+
- Each headline-only report is a duplicate for the reader.
|
|
97
|
+
- The parameters are valid only with EmbeddingGemma 2 and the vector text of round #3130.
|
|
98
|
+
- One clustering of a window of 1,001 reports takes 1.0 s, of 3,500 reports 9.6 s, and of 14,000 reports 175 s, through the `Collection` of `@zukhruf/embeddings`. The cost grows with the square of the window size (limitation 21).
|
|
99
|
+
|
|
100
|
+
## Decisions
|
|
101
|
+
|
|
102
|
+
Before you change a parameter, the similarity source or the model, read ADR 0001, ADR 0004, ADR 0005 and the limitations. A change needs a new research round on the evaluation sets.
|
|
103
|
+
|
|
104
|
+
Before you change a parameter, run `nx run event-detection:eval`, and run it again after the change. It scores the detector on the labelled sets and compares each score with the accepted designs ([eval/README.md](./eval/README.md)).
|
|
105
|
+
|
|
106
|
+
- [ADR 0001: The parameters are frozen by research round #3130](./docs/adr/0001-the-parameters-are-frozen-by-research-round-3130.md)
|
|
107
|
+
- [ADR 0002: The window is clustered again on each new publish day](./docs/adr/0002-the-window-is-clustered-again-on-each-new-publish-day.md)
|
|
108
|
+
- [ADR 0003: A headline-only report never joins another report](./docs/adr/0003-a-headline-only-report-never-joins-another-report.md)
|
|
109
|
+
- [ADR 0004: The similarity source is a port, and the parameters belong to one source](./docs/adr/0004-the-similarity-source-is-a-port-and-the-parameters-belong-to-one-source.md)
|
|
110
|
+
- [ADR 0005: Research round #3152 adds a same-publisher term and fits the weights again](./docs/adr/0005-research-round-3152-adds-a-same-publisher-term-and-fits-the-weights-again.md)
|
|
111
|
+
- [Limitations](./docs/limitations.md)
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Design } from './design.ts';
|
|
2
|
+
import type { Report } from './event-detector.ts';
|
|
3
|
+
import type { AnalysedReport } from './event-table.ts';
|
|
4
|
+
import { type IsTemplate } from './lead.ts';
|
|
5
|
+
export declare function analyse(report: Report, design: Pick<Design, 'lead' | 'vectorText'>, isTemplate?: IsTemplate): AnalysedReport;
|
|
6
|
+
//# sourceMappingURL=analyse.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyse.d.ts","sourceRoot":"","sources":["../src/analyse.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,KAAK,UAAU,EAAsB,MAAM,WAAW,CAAC;AAOhE,wBAAgB,OAAO,CACrB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,EAC3C,UAAU,CAAC,EAAE,UAAU,GACtB,cAAc,CAgBhB"}
|
package/dist/analyse.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { DAY_MS } from "./day.js";
|
|
2
|
+
import { leadOf, numberText } from "./lead.js";
|
|
3
|
+
import { numbersOf } from "./numbers.js";
|
|
4
|
+
// A report as a design reads it: its publish day, its vector text and the
|
|
5
|
+
// evidence of its pair score. The detector and the fitter (eval/fit) read
|
|
6
|
+
// reports with this one function. isTemplate: the template segments of the
|
|
7
|
+
// publisher of the report (src/templates.ts).
|
|
8
|
+
export function analyse(report, design, isTemplate) {
|
|
9
|
+
const lead = leadOf(report.title, report.body, design.lead, isTemplate);
|
|
10
|
+
const publishedAt = report.publishedAt.getTime();
|
|
11
|
+
return {
|
|
12
|
+
id: report.id,
|
|
13
|
+
publishedAt,
|
|
14
|
+
day: Math.floor(publishedAt / DAY_MS),
|
|
15
|
+
text: lead.title === null || lead.sentences.length === 0
|
|
16
|
+
? null
|
|
17
|
+
: vectorText(design.vectorText, lead.title, lead.sentences.join(' ')),
|
|
18
|
+
numbers: numbersOf(numberText(lead.title ?? '', report.body, design.lead, isTemplate)),
|
|
19
|
+
publisher: report.publisher ?? null,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
// One pass over the template, so a title that holds "{lead}" stays as it is.
|
|
23
|
+
function vectorText(template, title, lead) {
|
|
24
|
+
return template.replaceAll(/\{(title|lead)\}/gu, (_, name) => name === 'title' ? title : lead);
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=analyse.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyse.js","sourceRoot":"","sources":["../src/analyse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAIlC,OAAO,EAAmB,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,0EAA0E;AAC1E,0EAA0E;AAC1E,2EAA2E;AAC3E,8CAA8C;AAC9C,MAAM,UAAU,OAAO,CACrB,MAAc,EACd,MAA2C,EAC3C,UAAuB;IAEvB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;IACjD,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,WAAW;QACX,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;QACrC,IAAI,EACF,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAChD,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzE,OAAO,EAAE,SAAS,CAChB,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CACnE;QACD,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;KACpC,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,SAAS,UAAU,CAAC,QAAgB,EAAE,KAAa,EAAE,IAAY;IAC/D,OAAO,QAAQ,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAC3D,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAChC,CAAC;AACJ,CAAC"}
|
package/dist/day.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"day.d.ts","sourceRoot":"","sources":["../src/day.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,MAAM,WAAa,CAAC"}
|
package/dist/day.js
ADDED
package/dist/day.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"day.js","sourceRoot":"","sources":["../src/day.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,MAAM,CAAC,MAAM,MAAM,GAAG,UAAU,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"design-3152b.d.ts","sourceRoot":"","sources":["../src/design-3152b.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAQ1C,eAAO,MAAM,WAAW,EAAE,MAwCzB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Design 3152b. Round #3152 chose the pair score, and its second selection
|
|
2
|
+
// (3152b) chose the threshold with the CoNLL F0.5 objective. 3152b froze them
|
|
3
|
+
// in its plan.json before it scored its exam set. The lead rule, the vector
|
|
4
|
+
// text, the shortlist and the window are those of round #3130
|
|
5
|
+
// (docs/adr/0001). `nx run event-detection:fit -- 3152b` fits this design
|
|
6
|
+
// again from the training sets (docs/adr/0005).
|
|
7
|
+
export const design3152b = {
|
|
8
|
+
name: '3152b',
|
|
9
|
+
planSha256: 'dc26dbd8c26341ca4245552d017161307f30b79aef4bc5ba3a171a72a534c479',
|
|
10
|
+
embeddingModel: 'embeddinggemma-2',
|
|
11
|
+
lead: {
|
|
12
|
+
sentences: 2,
|
|
13
|
+
minWords: 5,
|
|
14
|
+
minLetterRatio: 0.6,
|
|
15
|
+
maxUrlShare: 0.5,
|
|
16
|
+
urlPattern: String.raw `(?:https?://|www\.)\S*`,
|
|
17
|
+
},
|
|
18
|
+
// The EmbeddingGemma document prompt.
|
|
19
|
+
vectorText: 'title: {title} | text: {title}\n{lead}',
|
|
20
|
+
shortlistSize: 80,
|
|
21
|
+
windowDays: 7,
|
|
22
|
+
threshold: 0.45,
|
|
23
|
+
pairScore: {
|
|
24
|
+
intercept: -5.577353541021158,
|
|
25
|
+
similarity: {
|
|
26
|
+
weight: 2.463189037261233,
|
|
27
|
+
mean: 0.6697853674594684,
|
|
28
|
+
sd: 0.0768108602613357,
|
|
29
|
+
},
|
|
30
|
+
agreeingNumbers: {
|
|
31
|
+
weight: 1.2184550356722605,
|
|
32
|
+
mean: 0.7080452669318389,
|
|
33
|
+
sd: 0.7146516956780061,
|
|
34
|
+
},
|
|
35
|
+
disagreeingNumbers: {
|
|
36
|
+
weight: -0.8987930339729852,
|
|
37
|
+
mean: 1.6871729914951807,
|
|
38
|
+
sd: 0.8093686065424462,
|
|
39
|
+
},
|
|
40
|
+
samePublisher: {
|
|
41
|
+
weight: -0.6864362601504641,
|
|
42
|
+
mean: 0.12460342227076507,
|
|
43
|
+
sd: 0.3302686927775403,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
//# sourceMappingURL=design-3152b.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"design-3152b.js","sourceRoot":"","sources":["../src/design-3152b.ts"],"names":[],"mappings":"AAEA,2EAA2E;AAC3E,8EAA8E;AAC9E,4EAA4E;AAC5E,8DAA8D;AAC9D,0EAA0E;AAC1E,gDAAgD;AAChD,MAAM,CAAC,MAAM,WAAW,GAAW;IACjC,IAAI,EAAE,OAAO;IACb,UAAU,EACR,kEAAkE;IACpE,cAAc,EAAE,kBAAkB;IAClC,IAAI,EAAE;QACJ,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,cAAc,EAAE,GAAG;QACnB,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAA,wBAAwB;KAC/C;IACD,sCAAsC;IACtC,UAAU,EAAE,wCAAwC;IACpD,aAAa,EAAE,EAAE;IACjB,UAAU,EAAE,CAAC;IACb,SAAS,EAAE,IAAI;IACf,SAAS,EAAE;QACT,SAAS,EAAE,CAAC,iBAAiB;QAC7B,UAAU,EAAE;YACV,MAAM,EAAE,iBAAiB;YACzB,IAAI,EAAE,kBAAkB;YACxB,EAAE,EAAE,kBAAkB;SACvB;QACD,eAAe,EAAE;YACf,MAAM,EAAE,kBAAkB;YAC1B,IAAI,EAAE,kBAAkB;YACxB,EAAE,EAAE,kBAAkB;SACvB;QACD,kBAAkB,EAAE;YAClB,MAAM,EAAE,CAAC,kBAAkB;YAC3B,IAAI,EAAE,kBAAkB;YACxB,EAAE,EAAE,kBAAkB;SACvB;QACD,aAAa,EAAE;YACb,MAAM,EAAE,CAAC,kBAAkB;YAC3B,IAAI,EAAE,mBAAmB;YACzB,EAAE,EAAE,kBAAkB;SACvB;KACF;CACF,CAAC"}
|
package/dist/design.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export type Term = {
|
|
2
|
+
readonly weight: number;
|
|
3
|
+
readonly mean: number;
|
|
4
|
+
readonly sd: number;
|
|
5
|
+
};
|
|
6
|
+
export type Design = {
|
|
7
|
+
readonly name: string;
|
|
8
|
+
readonly planSha256: string;
|
|
9
|
+
readonly embeddingModel: string;
|
|
10
|
+
readonly lead: {
|
|
11
|
+
readonly sentences: number;
|
|
12
|
+
readonly minWords: number;
|
|
13
|
+
readonly minLetterRatio: number;
|
|
14
|
+
readonly maxUrlShare: number;
|
|
15
|
+
readonly urlPattern: string;
|
|
16
|
+
readonly clean?: {
|
|
17
|
+
readonly templateReports: number | null;
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
readonly vectorText: string;
|
|
21
|
+
readonly shortlistSize: number;
|
|
22
|
+
readonly windowDays: number;
|
|
23
|
+
readonly threshold: number;
|
|
24
|
+
readonly pairScore: {
|
|
25
|
+
readonly intercept: number;
|
|
26
|
+
readonly similarity: Term;
|
|
27
|
+
readonly agreeingNumbers: Term;
|
|
28
|
+
readonly disagreeingNumbers: Term;
|
|
29
|
+
readonly samePublisher: Term;
|
|
30
|
+
readonly samePublisherSimilarity?: Term;
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
//# sourceMappingURL=design.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"design.d.ts","sourceRoot":"","sources":["../src/design.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,IAAI,GAAG;IACjB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB,CAAC;AAMF,MAAM,MAAM,MAAM,GAAG;IAEnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAGhC,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAE7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAK5B,QAAQ,CAAC,KAAK,CAAC,EAAE;YAKf,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;SACzC,CAAC;KACH,CAAC;IAGF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAG/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAG5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B,QAAQ,CAAC,SAAS,EAAE;QAClB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC;QAC1B,QAAQ,CAAC,eAAe,EAAE,IAAI,CAAC;QAC/B,QAAQ,CAAC,kBAAkB,EAAE,IAAI,CAAC;QAClC,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;QAI7B,QAAQ,CAAC,uBAAuB,CAAC,EAAE,IAAI,CAAC;KACzC,CAAC;CACH,CAAC"}
|
package/dist/design.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"design.js","sourceRoot":"","sources":["../src/design.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { Design } from './design.ts';
|
|
3
|
+
import type { Vectors } from './vectors.ts';
|
|
4
|
+
export type Report = {
|
|
5
|
+
readonly id: string;
|
|
6
|
+
readonly title: string;
|
|
7
|
+
readonly body: string;
|
|
8
|
+
readonly publishedAt: Date;
|
|
9
|
+
readonly publisher?: string;
|
|
10
|
+
};
|
|
11
|
+
export type Event = {
|
|
12
|
+
readonly id: number;
|
|
13
|
+
readonly members: readonly string[];
|
|
14
|
+
readonly firstSeen: Date;
|
|
15
|
+
readonly lastSeen: Date;
|
|
16
|
+
};
|
|
17
|
+
export type EventDetectorOptions = {
|
|
18
|
+
readonly database: DatabaseSync;
|
|
19
|
+
readonly vectors: Vectors;
|
|
20
|
+
readonly design: Design;
|
|
21
|
+
};
|
|
22
|
+
export declare class EventDetector {
|
|
23
|
+
#private;
|
|
24
|
+
constructor(options: EventDetectorOptions);
|
|
25
|
+
add(reports: readonly Report[]): Promise<void>;
|
|
26
|
+
events(options: {
|
|
27
|
+
readonly since: Date;
|
|
28
|
+
}): Event[];
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=event-detector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-detector.d.ts","sourceRoot":"","sources":["../src/event-detector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAQ1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,MAAM,MAAM,GAAG;IACnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC;IAI3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,KAAK,GAAG;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAEjC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAGhC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAAC;AAMF,qBAAa,aAAa;;gBAOZ,OAAO,EAAE,oBAAoB;IAYnC,GAAG,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDpD,MAAM,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAA;KAAE,GAAG,KAAK,EAAE;CAWnD"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { transaction } from '@zukhruf/sqlite-transaction';
|
|
2
|
+
import { analyse } from "./analyse.js";
|
|
3
|
+
import { DAY_MS } from "./day.js";
|
|
4
|
+
import { EventTable } from "./event-table.js";
|
|
5
|
+
import { assign } from "./grouping/assign.js";
|
|
6
|
+
import { recluster } from "./grouping/recluster.js";
|
|
7
|
+
import { PairScores } from "./grouping/scores.js";
|
|
8
|
+
import { Shortlist } from "./grouping/shortlist.js";
|
|
9
|
+
import { eventSchema } from "./schema.js";
|
|
10
|
+
import { Templates } from "./templates.js";
|
|
11
|
+
// Puts the reports of one real event into one event. Between two publish
|
|
12
|
+
// days, a report joins an event by online group average. When a report of a
|
|
13
|
+
// new publish day arrives, the detector first clusters the reports of the
|
|
14
|
+
// last 7 days again, with group average (docs/adr/0002).
|
|
15
|
+
export class EventDetector {
|
|
16
|
+
#database;
|
|
17
|
+
#vectors;
|
|
18
|
+
#design;
|
|
19
|
+
#table;
|
|
20
|
+
#templates;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
this.#database = options.database;
|
|
23
|
+
this.#vectors = options.vectors;
|
|
24
|
+
this.#design = options.design;
|
|
25
|
+
this.#table = new EventTable(this.#database);
|
|
26
|
+
this.#templates = new Templates(this.#database, this.#design.lead);
|
|
27
|
+
eventSchema.apply(this.#database);
|
|
28
|
+
}
|
|
29
|
+
// Gives each new report an event, in the order of the publish dates. Equal
|
|
30
|
+
// publish dates keep the order of the array. A report whose id the detector
|
|
31
|
+
// has already is skipped.
|
|
32
|
+
async add(reports) {
|
|
33
|
+
const unseen = reports.filter((report) => !this.#table.has(report.id));
|
|
34
|
+
this.#templates.observe(unseen);
|
|
35
|
+
const templates = new Map([...new Set(unseen.map((report) => report.publisher))].map((publisher) => [publisher, this.#templates.of(publisher)]));
|
|
36
|
+
const fresh = unseen
|
|
37
|
+
.map((report) => analyse(report, this.#design, templates.get(report.publisher)))
|
|
38
|
+
.toSorted((a, b) => a.publishedAt - b.publishedAt);
|
|
39
|
+
const withLead = fresh.flatMap(({ id, day, text }) => text === null ? [] : [{ id, text, publishedAt: new Date(day * DAY_MS) }]);
|
|
40
|
+
await this.#vectors.upsert(withLead);
|
|
41
|
+
transaction(this.#database, () => {
|
|
42
|
+
const design = this.#design;
|
|
43
|
+
const scores = new PairScores(this.#table, this.#vectors, design.pairScore);
|
|
44
|
+
let waiting = withLead.length;
|
|
45
|
+
for (const report of fresh) {
|
|
46
|
+
const shortlist = new Shortlist(this.#table, this.#vectors, design, waiting);
|
|
47
|
+
const current = this.#table.currentDay();
|
|
48
|
+
if (current !== undefined && report.day > current)
|
|
49
|
+
recluster(current, this.#table, shortlist, scores, design);
|
|
50
|
+
this.#table.advance(report.day);
|
|
51
|
+
assign(report, this.#table, shortlist, scores, design.threshold);
|
|
52
|
+
if (report.text !== null)
|
|
53
|
+
waiting--;
|
|
54
|
+
}
|
|
55
|
+
}, { mode: 'immediate' });
|
|
56
|
+
}
|
|
57
|
+
// The events with a report published at or after since, in the order of
|
|
58
|
+
// their first report.
|
|
59
|
+
events(options) {
|
|
60
|
+
const rows = this.#table.rows(options.since.getTime());
|
|
61
|
+
return [...Map.groupBy(rows, (row) => row.event).entries()].map(([id, members]) => ({
|
|
62
|
+
id,
|
|
63
|
+
members: members.map((member) => member.id),
|
|
64
|
+
firstSeen: new Date(members[0]?.publishedAt ?? 0),
|
|
65
|
+
lastSeen: new Date(members.at(-1)?.publishedAt ?? 0),
|
|
66
|
+
}));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=event-detector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-detector.js","sourceRoot":"","sources":["../src/event-detector.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAiC3C,yEAAyE;AACzE,4EAA4E;AAC5E,0EAA0E;AAC1E,yDAAyD;AACzD,MAAM,OAAO,aAAa;IACf,SAAS,CAAe;IACxB,QAAQ,CAAU;IAClB,OAAO,CAAS;IAChB,MAAM,CAAa;IACnB,UAAU,CAAY;IAE/B,YAAY,OAA6B;QACvC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,UAAU,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnE,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;IAED,2EAA2E;IAC3E,4EAA4E;IAC5E,0BAA0B;IAC1B,KAAK,CAAC,GAAG,CAAC,OAA0B;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CACxD,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAC1D,CACF,CAAC;QACF,MAAM,KAAK,GAAG,MAAM;aACjB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACd,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAC/D;aACA,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CACnD,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CACzE,CAAC;QACF,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAErC,WAAW,CACT,IAAI,CAAC,SAAS,EACd,GAAG,EAAE;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,EACb,MAAM,CAAC,SAAS,CACjB,CAAC;YACF,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC9B,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC3B,MAAM,SAAS,GAAG,IAAI,SAAS,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,EACb,MAAM,EACN,OAAO,CACR,CAAC;gBACF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBACzC,IAAI,OAAO,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,GAAG,OAAO;oBAC/C,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjE,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;oBAAE,OAAO,EAAE,CAAC;YACtC,CAAC;QACH,CAAC,EACD,EAAE,IAAI,EAAE,WAAW,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,sBAAsB;IACtB,MAAM,CAAC,OAAiC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAC7D,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClB,EAAE;YACF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,SAAS,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC;YACjD,QAAQ,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC;SACrD,CAAC,CACH,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { Evidence } from './pair-score.ts';
|
|
3
|
+
export type AnalysedReport = Evidence & {
|
|
4
|
+
readonly id: string;
|
|
5
|
+
readonly publishedAt: number;
|
|
6
|
+
readonly day: number;
|
|
7
|
+
readonly text: string | null;
|
|
8
|
+
};
|
|
9
|
+
export declare class EventTable {
|
|
10
|
+
#private;
|
|
11
|
+
constructor(database: DatabaseSync);
|
|
12
|
+
has(id: string): boolean;
|
|
13
|
+
eventOf(id: string): number;
|
|
14
|
+
members(event: number): string[];
|
|
15
|
+
evidenceOf(id: string): Evidence;
|
|
16
|
+
withLeadSince(day: number): string[];
|
|
17
|
+
insert(report: AnalysedReport, event: number): void;
|
|
18
|
+
move(id: string, event: number): void;
|
|
19
|
+
currentDay(): number | undefined;
|
|
20
|
+
advance(day: number): void;
|
|
21
|
+
newEvent(): number;
|
|
22
|
+
rows(since: number): {
|
|
23
|
+
id: string;
|
|
24
|
+
event: number;
|
|
25
|
+
publishedAt: number;
|
|
26
|
+
}[];
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=event-table.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-table.d.ts","sourceRoot":"","sources":["../src/event-table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG;IACtC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAGF,qBAAa,UAAU;;gBAGT,QAAQ,EAAE,YAAY;IAIlC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAQxB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAQ3B,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAShC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ;IAgBhC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IASpC,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAgBnD,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAMrC,UAAU,IAAI,MAAM,GAAG,SAAS;IAOhC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAS1B,QAAQ,IAAI,MAAM;IAUlB,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE;CAc1E"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// The rows of the detector in the caller's database.
|
|
2
|
+
export class EventTable {
|
|
3
|
+
#database;
|
|
4
|
+
constructor(database) {
|
|
5
|
+
this.#database = database;
|
|
6
|
+
}
|
|
7
|
+
has(id) {
|
|
8
|
+
return (this.#database
|
|
9
|
+
.prepare('SELECT 1 FROM "event_report" WHERE "id" = ?')
|
|
10
|
+
.get(id) !== undefined);
|
|
11
|
+
}
|
|
12
|
+
eventOf(id) {
|
|
13
|
+
const row = this.#database
|
|
14
|
+
.prepare('SELECT "event" FROM "event_report" WHERE "id" = ?')
|
|
15
|
+
.get(id);
|
|
16
|
+
if (row === undefined)
|
|
17
|
+
throw new Error(`No report has the id ${id}.`);
|
|
18
|
+
return Number(row.event);
|
|
19
|
+
}
|
|
20
|
+
members(event) {
|
|
21
|
+
return this.#database
|
|
22
|
+
.prepare('SELECT "id" FROM "event_report" WHERE "event" = ? ORDER BY "published_at", "id"')
|
|
23
|
+
.all(event)
|
|
24
|
+
.map((row) => String(row.id));
|
|
25
|
+
}
|
|
26
|
+
evidenceOf(id) {
|
|
27
|
+
const row = this.#database
|
|
28
|
+
.prepare('SELECT "numbers", "publisher" FROM "event_report" WHERE "id" = ?')
|
|
29
|
+
.get(id);
|
|
30
|
+
if (row === undefined)
|
|
31
|
+
throw new Error(`No report has the id ${id}.`);
|
|
32
|
+
const numbers = String(row.numbers);
|
|
33
|
+
return {
|
|
34
|
+
numbers: new Set(numbers === '' ? [] : numbers.split(' ')),
|
|
35
|
+
publisher: row.publisher === null ? null : String(row.publisher),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
// The reports with a lead, published on the day or later, in the order of
|
|
39
|
+
// their ids.
|
|
40
|
+
withLeadSince(day) {
|
|
41
|
+
return this.#database
|
|
42
|
+
.prepare('SELECT "id" FROM "event_report" WHERE "day" >= ? AND "has_lead" = 1 ORDER BY "id"')
|
|
43
|
+
.all(day)
|
|
44
|
+
.map((row) => String(row.id));
|
|
45
|
+
}
|
|
46
|
+
insert(report, event) {
|
|
47
|
+
this.#database
|
|
48
|
+
.prepare('INSERT INTO "event_report" ("id", "event", "published_at", "day", "has_lead", "numbers", "publisher") VALUES (?, ?, ?, ?, ?, ?, ?)')
|
|
49
|
+
.run(report.id, event, report.publishedAt, report.day, report.text === null ? 0 : 1, [...report.numbers].join(' '), report.publisher);
|
|
50
|
+
}
|
|
51
|
+
move(id, event) {
|
|
52
|
+
this.#database
|
|
53
|
+
.prepare('UPDATE "event_report" SET "event" = ? WHERE "id" = ?')
|
|
54
|
+
.run(event, id);
|
|
55
|
+
}
|
|
56
|
+
currentDay() {
|
|
57
|
+
const row = this.#database
|
|
58
|
+
.prepare('SELECT "current_day" FROM "event_clock" WHERE "id" = 1')
|
|
59
|
+
.get();
|
|
60
|
+
return row === undefined ? undefined : Number(row.current_day);
|
|
61
|
+
}
|
|
62
|
+
advance(day) {
|
|
63
|
+
this.#database
|
|
64
|
+
.prepare(`INSERT INTO "event_clock" ("id", "current_day", "next_event") VALUES (1, ?, 0)
|
|
65
|
+
ON CONFLICT ("id") DO UPDATE SET "current_day" = max("current_day", excluded."current_day")`)
|
|
66
|
+
.run(day);
|
|
67
|
+
}
|
|
68
|
+
newEvent() {
|
|
69
|
+
const row = this.#database
|
|
70
|
+
.prepare('UPDATE "event_clock" SET "next_event" = "next_event" + 1 WHERE "id" = 1 RETURNING "next_event" - 1 AS "event"')
|
|
71
|
+
.get();
|
|
72
|
+
if (row === undefined)
|
|
73
|
+
throw new Error('The clock has no row.');
|
|
74
|
+
return Number(row.event);
|
|
75
|
+
}
|
|
76
|
+
rows(since) {
|
|
77
|
+
return this.#database
|
|
78
|
+
.prepare(`SELECT "id", "event", "published_at" FROM "event_report"
|
|
79
|
+
WHERE "event" IN (SELECT "event" FROM "event_report" GROUP BY "event" HAVING max("published_at") >= ?)
|
|
80
|
+
ORDER BY "published_at", "id"`)
|
|
81
|
+
.all(since)
|
|
82
|
+
.map((row) => ({
|
|
83
|
+
id: String(row.id),
|
|
84
|
+
event: Number(row.event),
|
|
85
|
+
publishedAt: Number(row.published_at),
|
|
86
|
+
}));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=event-table.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-table.js","sourceRoot":"","sources":["../src/event-table.ts"],"names":[],"mappings":"AAaA,qDAAqD;AACrD,MAAM,OAAO,UAAU;IACZ,SAAS,CAAe;IAEjC,YAAY,QAAsB;QAChC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,CACL,IAAI,CAAC,SAAS;aACX,OAAO,CAAC,6CAA6C,CAAC;aACtD,GAAG,CAAC,EAAE,CAAC,KAAK,SAAS,CACzB,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,EAAU;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,mDAAmD,CAAC;aAC5D,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;QACtE,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,KAAa;QACnB,OAAO,IAAI,CAAC,SAAS;aAClB,OAAO,CACN,iFAAiF,CAClF;aACA,GAAG,CAAC,KAAK,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC;IAED,UAAU,CAAC,EAAU;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CACN,kEAAkE,CACnE;aACA,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO;YACL,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1D,SAAS,EAAE,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;SACjE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,aAAa;IACb,aAAa,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS;aAClB,OAAO,CACN,mFAAmF,CACpF;aACA,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,MAAsB,EAAE,KAAa;QAC1C,IAAI,CAAC,SAAS;aACX,OAAO,CACN,oIAAoI,CACrI;aACA,GAAG,CACF,MAAM,CAAC,EAAE,EACT,KAAK,EACL,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC5B,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7B,MAAM,CAAC,SAAS,CACjB,CAAC;IACN,CAAC;IAED,IAAI,CAAC,EAAU,EAAE,KAAa;QAC5B,IAAI,CAAC,SAAS;aACX,OAAO,CAAC,sDAAsD,CAAC;aAC/D,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACpB,CAAC;IAED,UAAU;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CAAC,wDAAwD,CAAC;aACjE,GAAG,EAAE,CAAC;QACT,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,IAAI,CAAC,SAAS;aACX,OAAO,CACN;oGAC4F,CAC7F;aACA,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,QAAQ;QACN,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS;aACvB,OAAO,CACN,+GAA+G,CAChH;aACA,GAAG,EAAE,CAAC;QACT,IAAI,GAAG,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAChE,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,KAAa;QAChB,OAAO,IAAI,CAAC,SAAS;aAClB,OAAO,CACN;;sCAE8B,CAC/B;aACA,GAAG,CAAC,KAAK,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YACxB,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;SACtC,CAAC,CAAC,CAAC;IACR,CAAC;CACF"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { AnalysedReport, EventTable } from '../event-table.ts';
|
|
2
|
+
import type { PairScores } from './scores.ts';
|
|
3
|
+
import type { Shortlist } from './shortlist.ts';
|
|
4
|
+
export declare function assign(report: AnalysedReport, table: EventTable, shortlist: Shortlist, scores: PairScores, threshold: number): void;
|
|
5
|
+
//# sourceMappingURL=assign.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assign.d.ts","sourceRoot":"","sources":["../../src/grouping/assign.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAMhD,wBAAgB,MAAM,CACpB,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,GAChB,IAAI,CAoBN"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Online group average: the report joins the event of its shortlist with the
|
|
2
|
+
// highest mean pair score over all members, when that mean is at least the
|
|
3
|
+
// threshold. Otherwise it starts a new event. A headline-only report always
|
|
4
|
+
// starts a new event (docs/adr/0003).
|
|
5
|
+
export function assign(report, table, shortlist, scores, threshold) {
|
|
6
|
+
let best;
|
|
7
|
+
let bestScore = -Infinity;
|
|
8
|
+
if (report.text !== null) {
|
|
9
|
+
scores.remember(report.id, report);
|
|
10
|
+
const candidates = [
|
|
11
|
+
...new Set(shortlist.of(report.id).map(({ id }) => table.eventOf(id))),
|
|
12
|
+
].toSorted((a, b) => a - b);
|
|
13
|
+
for (const event of candidates) {
|
|
14
|
+
const members = table.members(event);
|
|
15
|
+
const mean = members.reduce((sum, id) => sum + scores.score(report.id, id), 0) /
|
|
16
|
+
members.length;
|
|
17
|
+
if (mean >= threshold && mean > bestScore) {
|
|
18
|
+
best = event;
|
|
19
|
+
bestScore = mean;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
table.insert(report, best ?? table.newEvent());
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=assign.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assign.js","sourceRoot":"","sources":["../../src/grouping/assign.ts"],"names":[],"mappings":"AAIA,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,sCAAsC;AACtC,MAAM,UAAU,MAAM,CACpB,MAAsB,EACtB,KAAiB,EACjB,SAAoB,EACpB,MAAkB,EAClB,SAAiB;IAEjB,IAAI,IAAwB,CAAC;IAC7B,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;IAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG;YACjB,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;SACvE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,IAAI,GACR,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,MAAM,CAAC;YACjB,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;gBAC1C,IAAI,GAAG,KAAK,CAAC;gBACb,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;AACjD,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Design } from '../design.ts';
|
|
2
|
+
import type { EventTable } from '../event-table.ts';
|
|
3
|
+
import type { PairScores } from './scores.ts';
|
|
4
|
+
import type { Shortlist } from './shortlist.ts';
|
|
5
|
+
export declare function recluster(day: number, table: EventTable, shortlist: Shortlist, scores: PairScores, { threshold, windowDays }: Design): void;
|
|
6
|
+
//# sourceMappingURL=recluster.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recluster.d.ts","sourceRoot":"","sources":["../../src/grouping/recluster.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAOhD,wBAAgB,SAAS,CACvB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,UAAU,EAClB,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,MAAM,GAChC,IAAI,CA+BN"}
|