@dxos/pipeline-email 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +15 -0
- package/dist/lib/neutral/index.mjs +650 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +48 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/corpus/digest.d.ts +50 -0
- package/dist/types/src/corpus/digest.d.ts.map +1 -0
- package/dist/types/src/corpus/digest.test.d.ts +2 -0
- package/dist/types/src/corpus/digest.test.d.ts.map +1 -0
- package/dist/types/src/corpus/index.d.ts +6 -0
- package/dist/types/src/corpus/index.d.ts.map +1 -0
- package/dist/types/src/corpus/ledger.d.ts +33 -0
- package/dist/types/src/corpus/ledger.d.ts.map +1 -0
- package/dist/types/src/corpus/ledger.test.d.ts +2 -0
- package/dist/types/src/corpus/ledger.test.d.ts.map +1 -0
- package/dist/types/src/corpus/prompts.d.ts +21 -0
- package/dist/types/src/corpus/prompts.d.ts.map +1 -0
- package/dist/types/src/corpus/prompts.test.d.ts +2 -0
- package/dist/types/src/corpus/prompts.test.d.ts.map +1 -0
- package/dist/types/src/corpus/rollups.d.ts +21 -0
- package/dist/types/src/corpus/rollups.d.ts.map +1 -0
- package/dist/types/src/corpus/rollups.test.d.ts +2 -0
- package/dist/types/src/corpus/rollups.test.d.ts.map +1 -0
- package/dist/types/src/corpus/topics.d.ts +46 -0
- package/dist/types/src/corpus/topics.d.ts.map +1 -0
- package/dist/types/src/corpus/topics.test.d.ts +2 -0
- package/dist/types/src/corpus/topics.test.d.ts.map +1 -0
- package/dist/types/src/fact-pipeline.d.ts +26 -0
- package/dist/types/src/fact-pipeline.d.ts.map +1 -0
- package/dist/types/src/fact-pipeline.test.d.ts +2 -0
- package/dist/types/src/fact-pipeline.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +6 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/internal/fact-index.d.ts +8 -0
- package/dist/types/src/internal/fact-index.d.ts.map +1 -0
- package/dist/types/src/internal/fact-index.test.d.ts +2 -0
- package/dist/types/src/internal/fact-index.test.d.ts.map +1 -0
- package/dist/types/src/internal/threading.d.ts +4 -0
- package/dist/types/src/internal/threading.d.ts.map +1 -0
- package/dist/types/src/internal/threading.test.d.ts +2 -0
- package/dist/types/src/internal/threading.test.d.ts.map +1 -0
- package/dist/types/src/internal/threads.d.ts +9 -0
- package/dist/types/src/internal/threads.d.ts.map +1 -0
- package/dist/types/src/internal/threads.test.d.ts +2 -0
- package/dist/types/src/internal/threads.test.d.ts.map +1 -0
- package/dist/types/src/pipeline.d.ts +36 -0
- package/dist/types/src/pipeline.d.ts.map +1 -0
- package/dist/types/src/pipeline.test.d.ts +2 -0
- package/dist/types/src/pipeline.test.d.ts.map +1 -0
- package/dist/types/src/stages/context.d.ts +40 -0
- package/dist/types/src/stages/context.d.ts.map +1 -0
- package/dist/types/src/stages/extract-contacts.d.ts +6 -0
- package/dist/types/src/stages/extract-contacts.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts-commit.d.ts +13 -0
- package/dist/types/src/stages/extract-facts-commit.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts-commit.test.d.ts +2 -0
- package/dist/types/src/stages/extract-facts-commit.test.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts.d.ts +6 -0
- package/dist/types/src/stages/extract-facts.d.ts.map +1 -0
- package/dist/types/src/stages/extract-facts.test.d.ts +2 -0
- package/dist/types/src/stages/extract-facts.test.d.ts.map +1 -0
- package/dist/types/src/stages/facts.d.ts +7 -0
- package/dist/types/src/stages/facts.d.ts.map +1 -0
- package/dist/types/src/stages/facts.test.d.ts +2 -0
- package/dist/types/src/stages/facts.test.d.ts.map +1 -0
- package/dist/types/src/stages/index.d.ts +8 -0
- package/dist/types/src/stages/index.d.ts.map +1 -0
- package/dist/types/src/stages/stats.d.ts +6 -0
- package/dist/types/src/stages/stats.d.ts.map +1 -0
- package/dist/types/src/stages/stats.test.d.ts +2 -0
- package/dist/types/src/stages/stats.test.d.ts.map +1 -0
- package/dist/types/src/stages/summarize.d.ts +7 -0
- package/dist/types/src/stages/summarize.d.ts.map +1 -0
- package/dist/types/src/testing/email-extraction.bench.test.d.ts +2 -0
- package/dist/types/src/testing/email-extraction.bench.test.d.ts.map +1 -0
- package/dist/types/src/testing/email-fixtures.d.ts +25 -0
- package/dist/types/src/testing/email-fixtures.d.ts.map +1 -0
- package/dist/types/src/testing/email-pipeline.test.d.ts +2 -0
- package/dist/types/src/testing/email-pipeline.test.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +2 -0
- package/dist/types/src/testing/index.d.ts.map +1 -0
- package/dist/types/src/testing/parquet-email.test.d.ts +2 -0
- package/dist/types/src/testing/parquet-email.test.d.ts.map +1 -0
- package/dist/types/src/testing/parquet.d.ts +20 -0
- package/dist/types/src/testing/parquet.d.ts.map +1 -0
- package/dist/types/src/testing/parquet.test.d.ts +2 -0
- package/dist/types/src/testing/parquet.test.d.ts.map +1 -0
- package/dist/types/src/types/Thread.d.ts +18 -0
- package/dist/types/src/types/Thread.d.ts.map +1 -0
- package/dist/types/src/types/Thread.test.d.ts +2 -0
- package/dist/types/src/types/Thread.test.d.ts.map +1 -0
- package/dist/types/src/types/Topic.d.ts +12 -0
- package/dist/types/src/types/Topic.d.ts.map +1 -0
- package/dist/types/src/types/Topic.test.d.ts +2 -0
- package/dist/types/src/types/Topic.test.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +3 -0
- package/dist/types/src/types/index.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +58 -0
- package/src/corpus/digest.test.ts +78 -0
- package/src/corpus/digest.ts +92 -0
- package/src/corpus/index.ts +9 -0
- package/src/corpus/ledger.test.ts +85 -0
- package/src/corpus/ledger.ts +73 -0
- package/src/corpus/prompts.test.ts +24 -0
- package/src/corpus/prompts.ts +43 -0
- package/src/corpus/rollups.test.ts +50 -0
- package/src/corpus/rollups.ts +65 -0
- package/src/corpus/topics.test.ts +90 -0
- package/src/corpus/topics.ts +211 -0
- package/src/fact-pipeline.test.ts +63 -0
- package/src/fact-pipeline.ts +51 -0
- package/src/index.ts +9 -0
- package/src/internal/fact-index.test.ts +54 -0
- package/src/internal/fact-index.ts +43 -0
- package/src/internal/threading.test.ts +28 -0
- package/src/internal/threading.ts +27 -0
- package/src/internal/threads.test.ts +76 -0
- package/src/internal/threads.ts +73 -0
- package/src/pipeline.test.ts +74 -0
- package/src/pipeline.ts +76 -0
- package/src/stages/context.ts +32 -0
- package/src/stages/extract-contacts.ts +25 -0
- package/src/stages/extract-facts-commit.test.ts +75 -0
- package/src/stages/extract-facts-commit.ts +44 -0
- package/src/stages/extract-facts.test.ts +74 -0
- package/src/stages/extract-facts.ts +26 -0
- package/src/stages/facts.test.ts +40 -0
- package/src/stages/facts.ts +37 -0
- package/src/stages/index.ts +11 -0
- package/src/stages/stats.test.ts +38 -0
- package/src/stages/stats.ts +35 -0
- package/src/stages/summarize.ts +76 -0
- package/src/testing/email-extraction.bench.test.ts +117 -0
- package/src/testing/email-fixtures.ts +62 -0
- package/src/testing/email-pipeline.test.ts +483 -0
- package/src/testing/index.ts +8 -0
- package/src/testing/parquet-email.test.ts +114 -0
- package/src/testing/parquet.test.ts +73 -0
- package/src/testing/parquet.ts +82 -0
- package/src/types/Thread.test.ts +29 -0
- package/src/types/Thread.ts +26 -0
- package/src/types/Topic.test.ts +25 -0
- package/src/types/Topic.ts +20 -0
- package/src/types/index.ts +6 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"threading.d.ts","sourceRoot":"","sources":["../../../../src/internal/threading.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOtC,eAAO,MAAM,gBAAgB,YAAa,MAAM,KAAG,MAMlD,CAAC;AAKF,eAAO,MAAM,cAAc,YAAa,OAAO,CAAC,OAAO,KAAG,MAIzD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"threading.test.d.ts","sourceRoot":"","sources":["../../../../src/internal/threading.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Message } from '@dxos/types';
|
|
2
|
+
import { Thread } from '../types';
|
|
3
|
+
export type BuildThreadsOptions = {
|
|
4
|
+
readonly ownerEmail: string;
|
|
5
|
+
readonly now: string;
|
|
6
|
+
readonly stalePeriodMs?: number;
|
|
7
|
+
};
|
|
8
|
+
export declare const buildThreads: (messages: readonly Message.Message[], options: BuildThreadsOptions) => Thread[];
|
|
9
|
+
//# sourceMappingURL=threads.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"threads.d.ts","sourceRoot":"","sources":["../../../../src/internal/threads.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,MAAM,EAAoB,MAAM,UAAU,CAAC;AAKpD,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAeF,eAAO,MAAM,YAAY,aAAc,SAAS,OAAO,CAAC,OAAO,EAAE,WAAW,mBAAmB,KAAG,MAAM,EAwCvG,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"threads.test.d.ts","sourceRoot":"","sources":["../../../../src/internal/threads.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import * as Effect from 'effect/Effect';
|
|
2
|
+
import { type AiService } from '@dxos/ai';
|
|
3
|
+
import { type Database } from '@dxos/echo';
|
|
4
|
+
import { Message } from '@dxos/types';
|
|
5
|
+
import { type FactIndexer, type Stats, type Summary } from './stages';
|
|
6
|
+
import { type Thread } from './types';
|
|
7
|
+
export type EmailPipelineOptions = {
|
|
8
|
+
/** ECHO space database: extract-contacts persists Person/Organization, threads are added here. */
|
|
9
|
+
readonly db: Database.Database;
|
|
10
|
+
/** Persists one message's facts to the fact substrate (a closure over a `FactStore`). */
|
|
11
|
+
readonly indexFacts: FactIndexer;
|
|
12
|
+
/** The space owner's email, used to orient thread state (awaiting-mine / awaiting-theirs). */
|
|
13
|
+
readonly ownerEmail: string;
|
|
14
|
+
/** Wall clock used to derive thread staleness; injected so the assembly stays deterministic. */
|
|
15
|
+
readonly now: string;
|
|
16
|
+
};
|
|
17
|
+
export type EmailPipelineResult = {
|
|
18
|
+
readonly messages: Message.Message[];
|
|
19
|
+
readonly stats: Stats;
|
|
20
|
+
readonly summaries: ReadonlyArray<{
|
|
21
|
+
messageId: string;
|
|
22
|
+
summary: Summary;
|
|
23
|
+
}>;
|
|
24
|
+
readonly threads: Thread[];
|
|
25
|
+
};
|
|
26
|
+
export declare const EmailPipeline: {
|
|
27
|
+
/**
|
|
28
|
+
* Default assembly of the email stages over a batch of messages: summarize → extract-contacts →
|
|
29
|
+
* stats → extract-facts stream through with back pressure, then {@link buildThreads} groups the
|
|
30
|
+
* collected messages into canonical threads. Contacts and threads are persisted to `db`; the fact
|
|
31
|
+
* substrate is populated via `indexFacts`. Requires an `AiService` (summarize); the shared
|
|
32
|
+
* {@link EmailPipelineCtx} is constructed and provided internally.
|
|
33
|
+
*/
|
|
34
|
+
run: (messages: readonly Message.Message[], options: EmailPipelineOptions) => Effect.Effect<EmailPipelineResult, never, AiService.AiService>;
|
|
35
|
+
};
|
|
36
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,KAAK,EACV,KAAK,OAAO,EAMb,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,SAAS,CAAC;AAEtC,MAAM,MAAM,oBAAoB,GAAG;IACjC,kGAAkG;IAClG,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC/B,yFAAyF;IACzF,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC,8FAA8F;IAC9F,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gGAAgG;IAChG,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,eAAO,MAAM,aAAa;IACxB;;;;;;OAMG;IACH,GAAG,aACS,SAAS,OAAO,CAAC,OAAO,EAAE,WAC3B,oBAAoB,KAC5B,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC;CAoBlE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.test.d.ts","sourceRoot":"","sources":["../../../src/pipeline.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import * as Context from 'effect/Context';
|
|
2
|
+
import { type Database } from '@dxos/echo';
|
|
3
|
+
/** Summary produced by the summarize stage. */
|
|
4
|
+
export type Summary = {
|
|
5
|
+
readonly summary: string;
|
|
6
|
+
readonly isSpam: boolean;
|
|
7
|
+
readonly keywords: readonly string[];
|
|
8
|
+
};
|
|
9
|
+
/** Running tallies produced by the stats stage (mutable accumulator). */
|
|
10
|
+
export type Stats = {
|
|
11
|
+
from: Map<string, number>;
|
|
12
|
+
to: Map<string, number>;
|
|
13
|
+
total: number;
|
|
14
|
+
spam: number;
|
|
15
|
+
};
|
|
16
|
+
/** Per-message summary result keyed by messageId (for a summary view). */
|
|
17
|
+
export type SummaryResult = {
|
|
18
|
+
readonly summaries: ReadonlyArray<{
|
|
19
|
+
messageId: string;
|
|
20
|
+
summary: Summary;
|
|
21
|
+
}>;
|
|
22
|
+
};
|
|
23
|
+
export declare const emptyStats: () => Stats;
|
|
24
|
+
declare const EmailPipelineCtx_base: Context.TagClass<EmailPipelineCtx, "@dxos/pipeline-email/EmailPipelineCtx", {
|
|
25
|
+
readonly db: Database.Database;
|
|
26
|
+
readonly stats: Stats;
|
|
27
|
+
readonly summaries: Array<{
|
|
28
|
+
messageId: string;
|
|
29
|
+
summary: Summary;
|
|
30
|
+
}>;
|
|
31
|
+
}>;
|
|
32
|
+
/**
|
|
33
|
+
* Shared context threaded through the email stages via Effect's Requirements channel. `db` is the
|
|
34
|
+
* ECHO space database (browser-safe); `stats` and `summaries` are mutable accumulators read after
|
|
35
|
+
* the run. Callers provide it once at the pipeline edge (`Effect.provide`).
|
|
36
|
+
*/
|
|
37
|
+
export declare class EmailPipelineCtx extends EmailPipelineCtx_base {
|
|
38
|
+
}
|
|
39
|
+
export {};
|
|
40
|
+
//# sourceMappingURL=context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../src/stages/context.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAC;AAE1C,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,+CAA+C;AAC/C,MAAM,MAAM,OAAO,GAAG;IAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAEnH,yEAAyE;AACzE,MAAM,MAAM,KAAK,GAAG;IAAE,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAExG,0EAA0E;AAC1E,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;CAAE,CAAC;AAE3G,eAAO,MAAM,UAAU,QAAO,KAAgE,CAAC;;iBAU9E,QAAQ,CAAC,QAAQ;oBACd,KAAK;wBACD,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;;AAVtE;;;;GAIG;AACH,qBAAa,gBAAiB,SAAQ,qBAOnC;CAAG"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { Stage } from '@dxos/pipeline';
|
|
2
|
+
import { Message } from '@dxos/types';
|
|
3
|
+
import { EmailPipelineCtx } from './context';
|
|
4
|
+
/** Extract a Person (+ Organization) from the sender and persist to the ECHO space; pass message through. */
|
|
5
|
+
export declare const extractContactsStage: Stage.Stage<Message.Message, Message.Message, never, EmailPipelineCtx>;
|
|
6
|
+
//# sourceMappingURL=extract-contacts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-contacts.d.ts","sourceRoot":"","sources":["../../../../src/stages/extract-contacts.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,6GAA6G;AAC7G,eAAO,MAAM,oBAAoB,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,gBAAgB,CAWvG,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Stage } from '@dxos/pipeline';
|
|
2
|
+
import { type RDF } from '@dxos/pipeline-rdf';
|
|
3
|
+
import { Message } from '@dxos/types';
|
|
4
|
+
export type FactExtractor = (message: Message.Message) => Promise<RDF.Fact[]>;
|
|
5
|
+
export type FactUnit = {
|
|
6
|
+
readonly facts: RDF.Fact[];
|
|
7
|
+
/** Stable per-message id (`messageSource`); the dedup foreign id. */
|
|
8
|
+
readonly foreignId: string;
|
|
9
|
+
/** Monotonic cursor key. */
|
|
10
|
+
readonly key: number;
|
|
11
|
+
};
|
|
12
|
+
export declare const extractFactsUnitStage: (extract: FactExtractor) => Stage.Stage<Message.Message, FactUnit>;
|
|
13
|
+
//# sourceMappingURL=extract-facts-commit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-facts-commit.d.ts","sourceRoot":"","sources":["../../../../src/stages/extract-facts-commit.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAOtC,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;AAI9E,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,4BAA4B;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB,CAAC;AAMF,eAAO,MAAM,qBAAqB,YAAa,aAAa,KAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAOjG,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-facts-commit.test.d.ts","sourceRoot":"","sources":["../../../../src/stages/extract-facts-commit.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { Stage } from '@dxos/pipeline';
|
|
2
|
+
import { type RDF } from '@dxos/pipeline-rdf';
|
|
3
|
+
import { Message } from '@dxos/types';
|
|
4
|
+
export type FactIndexer = (message: Message.Message) => Promise<RDF.Fact[]>;
|
|
5
|
+
export declare const extractFactsStage: (indexFacts: FactIndexer) => Stage.Stage<Message.Message, Message.Message>;
|
|
6
|
+
//# sourceMappingURL=extract-facts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-facts.d.ts","sourceRoot":"","sources":["../../../../src/stages/extract-facts.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAItC,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;AAM5E,eAAO,MAAM,iBAAiB,eAAgB,WAAW,KAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAOrG,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-facts.test.d.ts","sourceRoot":"","sources":["../../../../src/stages/extract-facts.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type RDF } from '@dxos/pipeline-rdf';
|
|
2
|
+
import { Message } from '@dxos/types';
|
|
3
|
+
export declare const EMAIL_EXTRACTION_RULES: readonly string[];
|
|
4
|
+
export declare const EMAIL_EXTRACT_OPTIONS: RDF.ExtractOptions;
|
|
5
|
+
export declare const messageSource: (message: Message.Message) => string;
|
|
6
|
+
export declare const messageToDocument: (message: Message.Message) => RDF.ExtractDocument;
|
|
7
|
+
//# sourceMappingURL=facts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"facts.d.ts","sourceRoot":"","sources":["../../../../src/stages/facts.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAItC,eAAO,MAAM,sBAAsB,EAAE,SAAS,MAAM,EAInD,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,GAAG,CAAC,cAEvC,CAAC;AAKF,eAAO,MAAM,aAAa,YAAa,OAAO,CAAC,OAAO,KAAG,MAOxD,CAAC;AAEF,eAAO,MAAM,iBAAiB,YAAa,OAAO,CAAC,OAAO,KAAG,GAAG,CAAC,eAK/D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"facts.test.d.ts","sourceRoot":"","sources":["../../../../src/stages/facts.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/stages/index.ts"],"names":[],"mappings":"AAIA,cAAc,WAAW,CAAC;AAC1B,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { Stage } from '@dxos/pipeline';
|
|
2
|
+
import { Message } from '@dxos/types';
|
|
3
|
+
import { EmailPipelineCtx } from './context';
|
|
4
|
+
/** Pure-JS running tallies (senders, recipients, spam); pass message through. */
|
|
5
|
+
export declare const statsStage: Stage.Stage<Message.Message, Message.Message, never, EmailPipelineCtx>;
|
|
6
|
+
//# sourceMappingURL=stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../../src/stages/stats.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,iFAAiF;AACjF,eAAO,MAAM,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,gBAAgB,CAsB7F,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.d.ts","sourceRoot":"","sources":["../../../../src/stages/stats.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { AiService } from '@dxos/ai';
|
|
2
|
+
import { Stage } from '@dxos/pipeline';
|
|
3
|
+
import { Message } from '@dxos/types';
|
|
4
|
+
import { EmailPipelineCtx } from './context';
|
|
5
|
+
/** Summarize via AiService/LanguageModel; append a summary block and record spam/keywords + result. */
|
|
6
|
+
export declare const summarizeStage: Stage.Stage<Message.Message, Message.Message, never, EmailPipelineCtx | AiService.AiService>;
|
|
7
|
+
//# sourceMappingURL=summarize.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarize.d.ts","sourceRoot":"","sources":["../../../../src/stages/summarize.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAgB,OAAO,EAAE,MAAM,aAAa,CAAC;AAGpD,OAAO,EAAE,gBAAgB,EAAgB,MAAM,WAAW,CAAC;AA4B3D,uGAAuG;AACvG,eAAO,MAAM,cAAc,EAAE,KAAK,CAAC,KAAK,CACtC,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,OAAO,EACf,KAAK,EACL,gBAAgB,GAAG,SAAS,CAAC,SAAS,CA6BvC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"email-extraction.bench.test.d.ts","sourceRoot":"","sources":["../../../../src/testing/email-extraction.bench.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Message } from '@dxos/types';
|
|
2
|
+
import { type ParquetRow } from './parquet';
|
|
3
|
+
/**
|
|
4
|
+
* First shard of the Enron email dataset (https://huggingface.co/datasets/corbt/enron-emails). The
|
|
5
|
+
* HuggingFace CDN serves it with range requests + permissive CORS, so a browser reads only the first
|
|
6
|
+
* N rows (a few row-group byte ranges) rather than the whole ~185 MB file.
|
|
7
|
+
*/
|
|
8
|
+
export declare const DEFAULT_ENRON_URL = "https://huggingface.co/datasets/corbt/enron-emails/resolve/main/data/train-00000-of-00003.parquet";
|
|
9
|
+
export type LoadEnronOptions = {
|
|
10
|
+
/** Number of messages to read from the start of the dataset. */
|
|
11
|
+
readonly count?: number;
|
|
12
|
+
/** Parquet URL to read from; defaults to {@link DEFAULT_ENRON_URL}. */
|
|
13
|
+
readonly url?: string;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Load the first `count` (default 100) Enron messages directly from the dataset parquet over HTTP.
|
|
17
|
+
* Browser-safe: uses hyparquet's URL async buffer (range requests), so only the byte ranges the first
|
|
18
|
+
* rows need are fetched. Rows are mapped to ECHO Messages via {@link emailToMessage}.
|
|
19
|
+
*/
|
|
20
|
+
export declare const loadEnronMessages: ({ count, url }?: LoadEnronOptions) => Promise<Message.Message[]>;
|
|
21
|
+
/** Coerce a parquet timestamp cell (a `Date`, or a string/number) to an ISO string. */
|
|
22
|
+
export declare const asIso: (value: unknown) => string;
|
|
23
|
+
/** Map one email row (see the dataset's `dataset_info` schema) to a Message carrying the body as a text block. */
|
|
24
|
+
export declare const emailToMessage: (row: ParquetRow) => Message.Message;
|
|
25
|
+
//# sourceMappingURL=email-fixtures.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"email-fixtures.d.ts","sourceRoot":"","sources":["../../../../src/testing/email-fixtures.ts"],"names":[],"mappings":"AAMA,OAAO,EAAqB,OAAO,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,WAAW,CAAC;AAI5C;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,sGACuE,CAAC;AAEtG,MAAM,MAAM,gBAAgB,GAAG;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,oBAAoD,gBAAgB,KAAQ,OAAO,CAC/G,OAAO,CAAC,OAAO,EAAE,CAMlB,CAAC;AAEF,uFAAuF;AACvF,eAAO,MAAM,KAAK,UAAW,OAAO,KAAG,MACkC,CAAC;AAE1E,kHAAkH;AAClH,eAAO,MAAM,cAAc,QAAS,UAAU,KAAG,OAAO,CAAC,OAexD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"email-pipeline.test.d.ts","sourceRoot":"","sources":["../../../../src/testing/email-pipeline.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/testing/index.ts"],"names":[],"mappings":"AAIA,cAAc,kBAAkB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-email.test.d.ts","sourceRoot":"","sources":["../../../../src/testing/parquet-email.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import * as Stream from 'effect/Stream';
|
|
2
|
+
/** A decoded parquet row, keyed by column name. */
|
|
3
|
+
export type ParquetRow = Record<string, unknown>;
|
|
4
|
+
declare const ParquetReadError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
5
|
+
readonly _tag: "ParquetReadError";
|
|
6
|
+
} & Readonly<A>;
|
|
7
|
+
/** Failure opening, parsing, or decoding a parquet file; carries the offending path. */
|
|
8
|
+
export declare class ParquetReadError extends ParquetReadError_base<{
|
|
9
|
+
readonly file: string;
|
|
10
|
+
readonly cause: unknown;
|
|
11
|
+
}> {
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A pipeline source over a set of parquet files. Emits decoded rows as a back-pressured stream, one
|
|
15
|
+
* row group at a time, so decoded-row memory stays bounded regardless of file size. Files are read
|
|
16
|
+
* in the given order; within a file, rows follow row-group order.
|
|
17
|
+
*/
|
|
18
|
+
export declare const parquetSource: (files: readonly string[]) => Stream.Stream<ParquetRow, ParquetReadError>;
|
|
19
|
+
export {};
|
|
20
|
+
//# sourceMappingURL=parquet.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../../../src/testing/parquet.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AAIxC,mDAAmD;AACnD,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;;;;AAEjD,wFAAwF;AACxF,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB,CAAC;CAAG;AAIL;;;;GAIG;AACH,eAAO,MAAM,aAAa,UAAW,SAAS,MAAM,EAAE,KAAG,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,gBAAgB,CACpB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet.test.d.ts","sourceRoot":"","sources":["../../../../src/testing/parquet.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import * as Schema from 'effect/Schema';
|
|
2
|
+
import { Type } from '@dxos/echo';
|
|
3
|
+
export declare const ThreadState: Schema.Literal<["awaiting-mine", "awaiting-theirs", "resolved", "stalled"]>;
|
|
4
|
+
export type ThreadState = Schema.Schema.Type<typeof ThreadState>;
|
|
5
|
+
declare const Thread_base: Type.ObjClass<Thread, {
|
|
6
|
+
readonly threadId: string;
|
|
7
|
+
readonly subject: string;
|
|
8
|
+
readonly summary: string;
|
|
9
|
+
readonly state: "awaiting-mine" | "awaiting-theirs" | "resolved" | "stalled";
|
|
10
|
+
readonly participants: readonly string[];
|
|
11
|
+
readonly messageIds: readonly string[];
|
|
12
|
+
readonly openQuestions: readonly string[];
|
|
13
|
+
readonly actionItems: readonly string[];
|
|
14
|
+
}, {}>;
|
|
15
|
+
export declare class Thread extends Thread_base {
|
|
16
|
+
}
|
|
17
|
+
export {};
|
|
18
|
+
//# sourceMappingURL=Thread.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Thread.d.ts","sourceRoot":"","sources":["../../../../src/types/Thread.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAC;AAExC,OAAO,EAAO,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,eAAO,MAAM,WAAW,6EAA4E,CAAC;AACrG,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,WAAW,CAAC,CAAC;;;;;;;;;;;AAKjE,qBAAa,MAAO,SAAQ,WAW3B;CAAG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Thread.test.d.ts","sourceRoot":"","sources":["../../../../src/types/Thread.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Type } from '@dxos/echo';
|
|
2
|
+
declare const Topic_base: Type.ObjClass<Topic, {
|
|
3
|
+
readonly label: string;
|
|
4
|
+
readonly summary: string;
|
|
5
|
+
readonly threadIds: readonly string[];
|
|
6
|
+
readonly participants: readonly string[];
|
|
7
|
+
readonly keywords: readonly string[];
|
|
8
|
+
}, {}>;
|
|
9
|
+
export declare class Topic extends Topic_base {
|
|
10
|
+
}
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=Topic.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Topic.d.ts","sourceRoot":"","sources":["../../../../src/types/Topic.ts"],"names":[],"mappings":"AAMA,OAAO,EAAO,IAAI,EAAE,MAAM,YAAY,CAAC;;;;;;;;AAKvC,qBAAa,KAAM,SAAQ,UAQ1B;CAAG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Topic.test.d.ts","sourceRoot":"","sources":["../../../../src/types/Topic.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/types/index.ts"],"names":[],"mappings":"AAIA,cAAc,UAAU,CAAC;AACzB,cAAc,SAAS,CAAC"}
|