deso-ag 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +272 -0
- package/dist/fetchers/bluesky.d.ts +3 -0
- package/dist/fetchers/bluesky.d.ts.map +1 -0
- package/dist/fetchers/bluesky.js +154 -0
- package/dist/fetchers/bluesky.js.map +1 -0
- package/dist/fetchers/farcaster.d.ts +3 -0
- package/dist/fetchers/farcaster.d.ts.map +1 -0
- package/dist/fetchers/farcaster.js +149 -0
- package/dist/fetchers/farcaster.js.map +1 -0
- package/dist/fetchers/lens.d.ts +3 -0
- package/dist/fetchers/lens.d.ts.map +1 -0
- package/dist/fetchers/lens.js +125 -0
- package/dist/fetchers/lens.js.map +1 -0
- package/dist/fetchers/nostr.d.ts +3 -0
- package/dist/fetchers/nostr.d.ts.map +1 -0
- package/dist/fetchers/nostr.js +307 -0
- package/dist/fetchers/nostr.js.map +1 -0
- package/dist/formatters/output.d.ts +5 -0
- package/dist/formatters/output.d.ts.map +1 -0
- package/dist/formatters/output.js +225 -0
- package/dist/formatters/output.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +193 -0
- package/dist/index.js.map +1 -0
- package/dist/lib.d.ts +20 -0
- package/dist/lib.d.ts.map +1 -0
- package/dist/lib.js +71 -0
- package/dist/lib.js.map +1 -0
- package/dist/pipeline.d.ts +12 -0
- package/dist/pipeline.d.ts.map +1 -0
- package/dist/pipeline.js +136 -0
- package/dist/pipeline.js.map +1 -0
- package/dist/terms.d.ts +20 -0
- package/dist/terms.d.ts.map +1 -0
- package/dist/terms.js +158 -0
- package/dist/terms.js.map +1 -0
- package/dist/types.d.ts +67 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/search.d.ts +6 -0
- package/dist/utils/search.d.ts.map +1 -0
- package/dist/utils/search.js +13 -0
- package/dist/utils/search.js.map +1 -0
- package/dist/utils/time.d.ts +4 -0
- package/dist/utils/time.d.ts.map +1 -0
- package/dist/utils/time.js +24 -0
- package/dist/utils/time.js.map +1 -0
- package/package.json +62 -0
package/dist/pipeline.js
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { computeEngagementScore } from './formatters/output.js';
|
|
2
|
+
import { matchesQuery } from './utils/search.js';
|
|
3
|
+
export function normalizeContent(content) {
|
|
4
|
+
return content
|
|
5
|
+
.toLowerCase()
|
|
6
|
+
.replace(/https?:\/\/\S+/g, '')
|
|
7
|
+
.replace(/\s+/g, ' ')
|
|
8
|
+
.trim();
|
|
9
|
+
}
|
|
10
|
+
export function similarityRatio(a, b) {
|
|
11
|
+
const shorter = a.length <= b.length ? a : b;
|
|
12
|
+
const longer = a.length > b.length ? a : b;
|
|
13
|
+
if (longer.length === 0)
|
|
14
|
+
return 1;
|
|
15
|
+
let matches = 0;
|
|
16
|
+
for (let i = 0; i < shorter.length; i++) {
|
|
17
|
+
if (shorter[i] === longer[i])
|
|
18
|
+
matches++;
|
|
19
|
+
}
|
|
20
|
+
return matches / longer.length;
|
|
21
|
+
}
|
|
22
|
+
export function deduplicatePosts(posts) {
|
|
23
|
+
const result = [];
|
|
24
|
+
const normalizedCache = new Map();
|
|
25
|
+
for (const post of posts) {
|
|
26
|
+
const normalized = normalizeContent(post.content).slice(0, 200);
|
|
27
|
+
normalizedCache.set(post, normalized);
|
|
28
|
+
}
|
|
29
|
+
for (const post of posts) {
|
|
30
|
+
const normalized = normalizedCache.get(post);
|
|
31
|
+
let isDuplicate = false;
|
|
32
|
+
for (let i = 0; i < result.length; i++) {
|
|
33
|
+
const existing = result[i];
|
|
34
|
+
// Only deduplicate across different sources
|
|
35
|
+
if (existing.source === post.source)
|
|
36
|
+
continue;
|
|
37
|
+
const existingNormalized = normalizedCache.get(existing);
|
|
38
|
+
if (similarityRatio(normalized, existingNormalized) > 0.8) {
|
|
39
|
+
// Keep the one with higher engagement
|
|
40
|
+
if (computeEngagementScore(post) > computeEngagementScore(existing)) {
|
|
41
|
+
result[i] = post;
|
|
42
|
+
}
|
|
43
|
+
isDuplicate = true;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (!isDuplicate) {
|
|
48
|
+
result.push(post);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return result;
|
|
52
|
+
}
|
|
53
|
+
export function mergeResults(results, sort, query) {
|
|
54
|
+
const allPosts = [];
|
|
55
|
+
for (const result of results) {
|
|
56
|
+
allPosts.push(...result.posts);
|
|
57
|
+
}
|
|
58
|
+
// Deduplicate cross-source
|
|
59
|
+
const deduped = deduplicatePosts(allPosts);
|
|
60
|
+
// Sort
|
|
61
|
+
sortPosts(deduped, sort, query);
|
|
62
|
+
return deduped;
|
|
63
|
+
}
|
|
64
|
+
export function sortPosts(posts, sort, query) {
|
|
65
|
+
switch (sort) {
|
|
66
|
+
case 'recent':
|
|
67
|
+
posts.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
|
|
68
|
+
break;
|
|
69
|
+
case 'engagement':
|
|
70
|
+
posts.sort((a, b) => {
|
|
71
|
+
const diff = computeEngagementScore(b) - computeEngagementScore(a);
|
|
72
|
+
if (diff !== 0)
|
|
73
|
+
return diff;
|
|
74
|
+
return b.timestamp.getTime() - a.timestamp.getTime();
|
|
75
|
+
});
|
|
76
|
+
break;
|
|
77
|
+
case 'relevance':
|
|
78
|
+
if (query) {
|
|
79
|
+
posts.sort((a, b) => {
|
|
80
|
+
const aMatches = matchesQuery(a.content, a.tags || [], query);
|
|
81
|
+
const bMatches = matchesQuery(b.content, b.tags || [], query);
|
|
82
|
+
if (aMatches !== bMatches)
|
|
83
|
+
return aMatches ? -1 : 1;
|
|
84
|
+
const diff = computeEngagementScore(b) - computeEngagementScore(a);
|
|
85
|
+
if (diff !== 0)
|
|
86
|
+
return diff;
|
|
87
|
+
return b.timestamp.getTime() - a.timestamp.getTime();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// No query, fall back to engagement sort
|
|
92
|
+
posts.sort((a, b) => {
|
|
93
|
+
const diff = computeEngagementScore(b) - computeEngagementScore(a);
|
|
94
|
+
if (diff !== 0)
|
|
95
|
+
return diff;
|
|
96
|
+
return b.timestamp.getTime() - a.timestamp.getTime();
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
export function buildMeta(results, options) {
|
|
103
|
+
const totalPosts = results.reduce((sum, r) => sum + r.posts.length, 0);
|
|
104
|
+
return {
|
|
105
|
+
query: options.query,
|
|
106
|
+
sources: results.map(r => ({
|
|
107
|
+
name: r.source,
|
|
108
|
+
count: r.posts.length,
|
|
109
|
+
error: r.error,
|
|
110
|
+
})),
|
|
111
|
+
timeframe: options.timeframe,
|
|
112
|
+
fetchedAt: new Date().toISOString(),
|
|
113
|
+
totalPosts,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
export function parseSources(input) {
|
|
117
|
+
const valid = ['farcaster', 'lens', 'nostr', 'bluesky'];
|
|
118
|
+
const sources = input.split(',').map(s => s.trim().toLowerCase());
|
|
119
|
+
return sources.filter((s) => valid.includes(s));
|
|
120
|
+
}
|
|
121
|
+
export function parseTimeframe(input) {
|
|
122
|
+
const valid = ['24h', '48h', 'week'];
|
|
123
|
+
const tf = input.toLowerCase();
|
|
124
|
+
return valid.includes(tf) ? tf : '24h';
|
|
125
|
+
}
|
|
126
|
+
export function parseFormat(input) {
|
|
127
|
+
const valid = ['json', 'markdown', 'summary', 'compact'];
|
|
128
|
+
const fmt = input.toLowerCase();
|
|
129
|
+
return valid.includes(fmt) ? fmt : 'markdown';
|
|
130
|
+
}
|
|
131
|
+
export function parseSortOrder(input) {
|
|
132
|
+
const valid = ['engagement', 'recent', 'relevance'];
|
|
133
|
+
const sort = input.toLowerCase();
|
|
134
|
+
return valid.includes(sort) ? sort : 'engagement';
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,OAAO,OAAO;SACX,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,CAAS,EAAE,CAAS;IAClD,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,MAAM,eAAe,GAAG,IAAI,GAAG,EAAgB,CAAC;IAEhD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAChE,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAC9C,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3B,4CAA4C;YAC5C,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;gBAAE,SAAS;YAE9C,MAAM,kBAAkB,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YAC1D,IAAI,eAAe,CAAC,UAAU,EAAE,kBAAkB,CAAC,GAAG,GAAG,EAAE,CAAC;gBAC1D,sCAAsC;gBACtC,IAAI,sBAAsB,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;gBACnB,CAAC;gBACD,WAAW,GAAG,IAAI,CAAC;gBACnB,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAsB,EAAE,IAAe,EAAE,KAAc;IAClF,MAAM,QAAQ,GAAW,EAAE,CAAC;IAE5B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,2BAA2B;IAC3B,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAE3C,OAAO;IACP,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAEhC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,IAAe,EAAE,KAAc;IACtE,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACpE,MAAM;QACR,KAAK,YAAY;YACf,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAClB,MAAM,IAAI,GAAG,sBAAsB,CAAC,CAAC,CAAC,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,IAAI,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBAC5B,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACvD,CAAC,CAAC,CAAC;YACH,MAAM;QACR,KAAK,WAAW;YACd,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBAClB,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9D,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9D,IAAI,QAAQ,KAAK,QAAQ;wBAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpD,MAAM,IAAI,GAAG,sBAAsB,CAAC,CAAC,CAAC,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;oBACnE,IAAI,IAAI,KAAK,CAAC;wBAAE,OAAO,IAAI,CAAC;oBAC5B,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACvD,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBAClB,MAAM,IAAI,GAAG,sBAAsB,CAAC,CAAC,CAAC,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;oBACnE,IAAI,IAAI,KAAK,CAAC;wBAAE,OAAO,IAAI,CAAC;oBAC5B,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACvD,CAAC,CAAC,CAAC;YACL,CAAC;YACD,MAAM;IACV,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAsB,EAAE,OAAsB;IACtE,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvE,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,EAAE,CAAC,CAAC,MAAM;YACd,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM;YACrB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC;QACH,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,MAAM,KAAK,GAAa,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAElE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAW,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,KAAK,GAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,EAAe,CAAC;IAE5C,OAAO,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,KAAK,GAAmB,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzE,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,EAAkB,CAAC;IAEhD,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,KAAK,GAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,EAAe,CAAC;IAE9C,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC;AACpD,CAAC"}
|
package/dist/terms.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Post, Term, TermsResult } from './types.js';
|
|
2
|
+
export declare const STOP_WORDS: Set<string>;
|
|
3
|
+
/**
|
|
4
|
+
* Tokenize text into unigrams and bigrams.
|
|
5
|
+
* Returns arrays of tokens (lowercased strings).
|
|
6
|
+
*/
|
|
7
|
+
export declare function tokenize(text: string): {
|
|
8
|
+
unigrams: string[];
|
|
9
|
+
bigrams: string[];
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Extract top N terms from a list of posts using engagement-weighted
|
|
13
|
+
* word/bigram frequency analysis.
|
|
14
|
+
*/
|
|
15
|
+
export declare function extractTerms(posts: Post[], topN?: number): Term[];
|
|
16
|
+
/**
|
|
17
|
+
* Extract terms grouped by source + overall.
|
|
18
|
+
*/
|
|
19
|
+
export declare function extractTermsBySource(posts: Post[], topN?: number, timeframe?: string): TermsResult;
|
|
20
|
+
//# sourceMappingURL=terms.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"terms.d.ts","sourceRoot":"","sources":["../src/terms.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,IAAI,EAAU,IAAI,EAAe,WAAW,EAAE,MAAM,YAAY,CAAC;AAE/E,eAAO,MAAM,UAAU,aA0BrB,CAAC;AAkBH;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAchF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,GAAE,MAAU,GAAG,IAAI,EAAE,CA+EpE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,IAAI,EAAE,EACb,IAAI,GAAE,MAAU,EAChB,SAAS,GAAE,MAAc,GACxB,WAAW,CA4Bb"}
|
package/dist/terms.js
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { normalizeContent } from './pipeline.js';
|
|
2
|
+
import { computeEngagementScore } from './formatters/output.js';
|
|
3
|
+
export const STOP_WORDS = new Set([
|
|
4
|
+
'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have', 'i',
|
|
5
|
+
'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you', 'do', 'at',
|
|
6
|
+
'this', 'but', 'his', 'by', 'from', 'they', 'we', 'say', 'her',
|
|
7
|
+
'she', 'or', 'an', 'will', 'my', 'one', 'all', 'would', 'there',
|
|
8
|
+
'their', 'what', 'so', 'up', 'out', 'if', 'about', 'who', 'get',
|
|
9
|
+
'which', 'go', 'me', 'when', 'make', 'can', 'like', 'time', 'no',
|
|
10
|
+
'just', 'him', 'know', 'take', 'people', 'into', 'year', 'your',
|
|
11
|
+
'good', 'some', 'could', 'them', 'see', 'other', 'than', 'then',
|
|
12
|
+
'now', 'look', 'only', 'come', 'its', 'over', 'think', 'also',
|
|
13
|
+
'back', 'after', 'use', 'two', 'how', 'our', 'work', 'first',
|
|
14
|
+
'well', 'way', 'even', 'new', 'want', 'because', 'any', 'these',
|
|
15
|
+
'give', 'day', 'most', 'us', 'are', 'has', 'was', 'been', 'is',
|
|
16
|
+
'am', 'were', 'did', 'had', 'does', 'being', 'got', 'really',
|
|
17
|
+
'very', 'much', 'more', 'too', 'still', 'own', 'here', 'why',
|
|
18
|
+
'don', 'didn', 'isn', 'aren', 'won', 'wouldn', 'shouldn', 'couldn',
|
|
19
|
+
'let', 'thing', 'things', 'lot', 'going', 'need', 'right', 'big',
|
|
20
|
+
'long', 'man', 'old', 'great', 'little', 'sure', 'keep', 'should',
|
|
21
|
+
'those', 'made', 'said', 'lol', 'yes', 'yeah', 'nah', 'okay', 'actually',
|
|
22
|
+
'every', 'already', 'many', 'may', 'might', 'must', 'though', 'through',
|
|
23
|
+
'while', 'where', 'down', 'off', 'been', 'before', 'between', 'both',
|
|
24
|
+
'each', 'few', 'same', 'such', 'since', 'never', 'always', 'last',
|
|
25
|
+
'again', 'another', 'around', 'part', 'put', 'set', 'per', 'try',
|
|
26
|
+
'doing', 'done', 'getting', 'making', 'something', 'anything', 'everything',
|
|
27
|
+
'nothing', 'someone', 'anyone', 'everyone', 'nobody', 'gonna', 'wanna',
|
|
28
|
+
'gotta', 'ima', 'imo', 'tbh', 'idk',
|
|
29
|
+
]);
|
|
30
|
+
/**
|
|
31
|
+
* Strip punctuation and possessives from normalized text.
|
|
32
|
+
*/
|
|
33
|
+
function stripPunctuation(text) {
|
|
34
|
+
return text
|
|
35
|
+
.replace(/'s\b/g, '') // possessives
|
|
36
|
+
.replace(/[^\w\s]/g, ' ') // punctuation → space
|
|
37
|
+
.replace(/\s+/g, ' ')
|
|
38
|
+
.trim();
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Tokenize text into unigrams and bigrams.
|
|
42
|
+
* Returns arrays of tokens (lowercased strings).
|
|
43
|
+
*/
|
|
44
|
+
export function tokenize(text) {
|
|
45
|
+
const clean = stripPunctuation(text);
|
|
46
|
+
const words = clean.split(' ').filter(w => w.length >= 3);
|
|
47
|
+
const unigrams = words.filter(w => !STOP_WORDS.has(w));
|
|
48
|
+
const bigrams = [];
|
|
49
|
+
for (let i = 0; i < words.length - 1; i++) {
|
|
50
|
+
// Skip bigrams where both words are stop words
|
|
51
|
+
if (STOP_WORDS.has(words[i]) && STOP_WORDS.has(words[i + 1]))
|
|
52
|
+
continue;
|
|
53
|
+
bigrams.push(`${words[i]} ${words[i + 1]}`);
|
|
54
|
+
}
|
|
55
|
+
return { unigrams, bigrams };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Extract top N terms from a list of posts using engagement-weighted
|
|
59
|
+
* word/bigram frequency analysis.
|
|
60
|
+
*/
|
|
61
|
+
export function extractTerms(posts, topN = 3) {
|
|
62
|
+
const tokenWeights = new Map();
|
|
63
|
+
for (const post of posts) {
|
|
64
|
+
const normalized = normalizeContent(post.content);
|
|
65
|
+
// Skip short posts (noise like "gm")
|
|
66
|
+
if (normalized.length < 10)
|
|
67
|
+
continue;
|
|
68
|
+
const engagementScore = computeEngagementScore(post);
|
|
69
|
+
const weight = 1 + Math.log2(1 + engagementScore);
|
|
70
|
+
const { unigrams, bigrams } = tokenize(normalized);
|
|
71
|
+
// Include post.tags as additional unigram tokens
|
|
72
|
+
const tagTokens = (post.tags || []).map(t => t.toLowerCase()).filter(t => t.length >= 3);
|
|
73
|
+
const allUnigrams = [...unigrams, ...tagTokens];
|
|
74
|
+
// Accumulate unigrams
|
|
75
|
+
for (const token of allUnigrams) {
|
|
76
|
+
const entry = tokenWeights.get(token) || { weight: 0, postIds: new Set() };
|
|
77
|
+
entry.weight += weight;
|
|
78
|
+
entry.postIds.add(post.id);
|
|
79
|
+
tokenWeights.set(token, entry);
|
|
80
|
+
}
|
|
81
|
+
// Accumulate bigrams
|
|
82
|
+
for (const token of bigrams) {
|
|
83
|
+
const entry = tokenWeights.get(token) || { weight: 0, postIds: new Set() };
|
|
84
|
+
entry.weight += weight;
|
|
85
|
+
entry.postIds.add(post.id);
|
|
86
|
+
tokenWeights.set(token, entry);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// Threshold: require >= 2 distinct posts
|
|
90
|
+
const filtered = [];
|
|
91
|
+
for (const [token, entry] of tokenWeights) {
|
|
92
|
+
if (entry.postIds.size >= 2) {
|
|
93
|
+
filtered.push([token, entry]);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Sort by score descending, prefer bigrams when scores are equal
|
|
97
|
+
filtered.sort((a, b) => {
|
|
98
|
+
const diff = b[1].weight - a[1].weight;
|
|
99
|
+
if (diff !== 0)
|
|
100
|
+
return diff;
|
|
101
|
+
// Prefer bigrams over unigrams at equal weight (more specific)
|
|
102
|
+
const aIsBigram = a[0].includes(' ') ? 1 : 0;
|
|
103
|
+
const bIsBigram = b[0].includes(' ') ? 1 : 0;
|
|
104
|
+
return bIsBigram - aIsBigram;
|
|
105
|
+
});
|
|
106
|
+
// Deduplicate: if a bigram ranks higher, suppress its constituent unigrams
|
|
107
|
+
const result = [];
|
|
108
|
+
const suppressed = new Set();
|
|
109
|
+
for (const [token, entry] of filtered) {
|
|
110
|
+
if (suppressed.has(token))
|
|
111
|
+
continue;
|
|
112
|
+
result.push({
|
|
113
|
+
token,
|
|
114
|
+
score: Math.round(entry.weight * 10) / 10,
|
|
115
|
+
postCount: entry.postIds.size,
|
|
116
|
+
});
|
|
117
|
+
// If this is a bigram, suppress its constituent unigrams
|
|
118
|
+
if (token.includes(' ')) {
|
|
119
|
+
const parts = token.split(' ');
|
|
120
|
+
for (const part of parts) {
|
|
121
|
+
suppressed.add(part);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (result.length >= topN)
|
|
125
|
+
break;
|
|
126
|
+
}
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Extract terms grouped by source + overall.
|
|
131
|
+
*/
|
|
132
|
+
export function extractTermsBySource(posts, topN = 3, timeframe = '24h') {
|
|
133
|
+
// Group by source
|
|
134
|
+
const bySource = new Map();
|
|
135
|
+
for (const post of posts) {
|
|
136
|
+
const existing = bySource.get(post.source) || [];
|
|
137
|
+
existing.push(post);
|
|
138
|
+
bySource.set(post.source, existing);
|
|
139
|
+
}
|
|
140
|
+
const sourceTerms = [];
|
|
141
|
+
for (const [source, sourcePosts] of bySource) {
|
|
142
|
+
sourceTerms.push({
|
|
143
|
+
source,
|
|
144
|
+
postCount: sourcePosts.length,
|
|
145
|
+
terms: extractTerms(sourcePosts, topN),
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
// Sort sources in a consistent order
|
|
149
|
+
const sourceOrder = ['farcaster', 'lens', 'nostr', 'bluesky'];
|
|
150
|
+
sourceTerms.sort((a, b) => sourceOrder.indexOf(a.source) - sourceOrder.indexOf(b.source));
|
|
151
|
+
return {
|
|
152
|
+
bySource: sourceTerms,
|
|
153
|
+
overall: extractTerms(posts, topN),
|
|
154
|
+
timeframe,
|
|
155
|
+
analyzedAt: new Date().toISOString(),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=terms.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"terms.js","sourceRoot":"","sources":["../src/terms.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAGhE,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IAChC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG;IAC9D,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;IAC/D,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IAC9D,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO;IAC/D,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAC/D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI;IAChE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC/D,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IAC/D,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAC7D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC5D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO;IAC/D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI;IAC9D,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ;IAC5D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IAC5D,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ;IAClE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;IAChE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ;IACjE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU;IACxE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS;IACvE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM;IACpE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IACjE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAChE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY;IAC3E,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO;IACtE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;CACpC,CAAC,CAAC;AAOH;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,IAAI;SACR,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAM,cAAc;SACxC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAE,sBAAsB;SAChD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAE1D,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,+CAA+C;QAC/C,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QACvE,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,OAAe,CAAC;IAC1D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElD,qCAAqC;QACrC,IAAI,UAAU,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS;QAErC,MAAM,eAAe,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC;QAElD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEnD,iDAAiD;QACjD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;QAEzF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,SAAS,CAAC,CAAC;QAEhD,sBAAsB;QACtB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAU,EAAE,CAAC;YACnF,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YACvB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,qBAAqB;QACrB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAU,EAAE,CAAC;YACnF,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;YACvB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3B,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACvC,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5B,+DAA+D;QAC/D,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAErC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAEpC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,EAAE;YACzC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;SAC9B,CAAC,CAAC;QAEH,yDAAyD;QACzD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI;YAAE,MAAM;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAa,EACb,OAAe,CAAC,EAChB,YAAoB,KAAK;IAEzB,kBAAkB;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,WAAW,GAAkB,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC7C,WAAW,CAAC,IAAI,CAAC;YACf,MAAM;YACN,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,KAAK,EAAE,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC;SACvC,CAAC,CAAC;IACL,CAAC;IAED,qCAAqC;IACrC,MAAM,WAAW,GAAa,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACxE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAE1F,OAAO;QACL,QAAQ,EAAE,WAAW;QACrB,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC;QAClC,SAAS;QACT,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
export type Source = 'farcaster' | 'lens' | 'nostr' | 'bluesky';
|
|
2
|
+
export type Timeframe = '24h' | '48h' | 'week';
|
|
3
|
+
export type OutputFormat = 'json' | 'markdown' | 'summary' | 'compact';
|
|
4
|
+
export type SortOrder = 'engagement' | 'recent' | 'relevance';
|
|
5
|
+
export interface Post {
|
|
6
|
+
id: string;
|
|
7
|
+
source: Source;
|
|
8
|
+
author: {
|
|
9
|
+
username: string;
|
|
10
|
+
displayName?: string;
|
|
11
|
+
profileUrl?: string;
|
|
12
|
+
};
|
|
13
|
+
content: string;
|
|
14
|
+
timestamp: Date;
|
|
15
|
+
url?: string;
|
|
16
|
+
engagement?: {
|
|
17
|
+
likes?: number;
|
|
18
|
+
reposts?: number;
|
|
19
|
+
replies?: number;
|
|
20
|
+
};
|
|
21
|
+
channel?: string;
|
|
22
|
+
tags?: string[];
|
|
23
|
+
}
|
|
24
|
+
export interface SearchOptions {
|
|
25
|
+
query?: string;
|
|
26
|
+
sources: Source[];
|
|
27
|
+
timeframe: Timeframe;
|
|
28
|
+
channel?: string;
|
|
29
|
+
limit?: number;
|
|
30
|
+
sort?: SortOrder;
|
|
31
|
+
}
|
|
32
|
+
export interface FetchResult {
|
|
33
|
+
posts: Post[];
|
|
34
|
+
source: Source;
|
|
35
|
+
error?: string;
|
|
36
|
+
}
|
|
37
|
+
export interface AggregateResult {
|
|
38
|
+
posts: Post[];
|
|
39
|
+
meta: {
|
|
40
|
+
query?: string;
|
|
41
|
+
sources: {
|
|
42
|
+
name: Source;
|
|
43
|
+
count: number;
|
|
44
|
+
error?: string;
|
|
45
|
+
}[];
|
|
46
|
+
timeframe: string;
|
|
47
|
+
fetchedAt: string;
|
|
48
|
+
totalPosts: number;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export interface Term {
|
|
52
|
+
token: string;
|
|
53
|
+
score: number;
|
|
54
|
+
postCount: number;
|
|
55
|
+
}
|
|
56
|
+
export interface SourceTerms {
|
|
57
|
+
source: Source;
|
|
58
|
+
postCount: number;
|
|
59
|
+
terms: Term[];
|
|
60
|
+
}
|
|
61
|
+
export interface TermsResult {
|
|
62
|
+
bySource: SourceTerms[];
|
|
63
|
+
overall: Term[];
|
|
64
|
+
timeframe: string;
|
|
65
|
+
analyzedAt: string;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;AAChE,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;AAC/C,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;AACvE,MAAM,MAAM,SAAS,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE9D,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE;QACX,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,IAAI,EAAE;QACJ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC3D,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,MAAM,WAAW,IAAI;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,OAAO,EAAE,IAAI,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/utils/search.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAUpF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-word search with AND semantics.
|
|
3
|
+
* All terms must match (case-insensitive) against content or tags.
|
|
4
|
+
*/
|
|
5
|
+
export function matchesQuery(content, tags, query) {
|
|
6
|
+
const terms = query.toLowerCase().split(/\s+/).filter(t => t.length > 0);
|
|
7
|
+
if (terms.length === 0)
|
|
8
|
+
return true;
|
|
9
|
+
const lowerContent = content.toLowerCase();
|
|
10
|
+
const lowerTags = tags.map(t => t.toLowerCase());
|
|
11
|
+
return terms.every(term => lowerContent.includes(term) || lowerTags.some(tag => tag.includes(term)));
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/utils/search.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,KAAa;IACzE,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAEjD,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CACxB,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CACzE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"time.d.ts","sourceRoot":"","sources":["../../src/utils/time.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,CAY7D;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAOhD"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export function getTimeframeCutoff(timeframe) {
|
|
2
|
+
const now = new Date();
|
|
3
|
+
switch (timeframe) {
|
|
4
|
+
case '24h':
|
|
5
|
+
return new Date(now.getTime() - 24 * 60 * 60 * 1000);
|
|
6
|
+
case '48h':
|
|
7
|
+
return new Date(now.getTime() - 48 * 60 * 60 * 1000);
|
|
8
|
+
case 'week':
|
|
9
|
+
return new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
|
|
10
|
+
default:
|
|
11
|
+
return new Date(now.getTime() - 24 * 60 * 60 * 1000);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export function formatTimeAgo(date) {
|
|
15
|
+
const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
|
|
16
|
+
if (seconds < 60)
|
|
17
|
+
return `${seconds}s ago`;
|
|
18
|
+
if (seconds < 3600)
|
|
19
|
+
return `${Math.floor(seconds / 60)}m ago`;
|
|
20
|
+
if (seconds < 86400)
|
|
21
|
+
return `${Math.floor(seconds / 3600)}h ago`;
|
|
22
|
+
return `${Math.floor(seconds / 86400)}d ago`;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=time.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"time.js","sourceRoot":"","sources":["../../src/utils/time.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,kBAAkB,CAAC,SAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,KAAK;YACR,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QACvD,KAAK,KAAK;YACR,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QACvD,KAAK,MAAM;YACT,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC3D;YACE,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAU;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAEjE,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,GAAG,OAAO,OAAO,CAAC;IAC3C,IAAI,OAAO,GAAG,IAAI;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC;IAC9D,IAAI,OAAO,GAAG,KAAK;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IACjE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;AAC/C,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "deso-ag",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "CLI tool for aggregating posts from decentralized social protocols (Farcaster, Lens, Nostr, Bluesky)",
|
|
5
|
+
"main": "dist/lib.js",
|
|
6
|
+
"bin": {
|
|
7
|
+
"deso-ag": "./dist/index.js"
|
|
8
|
+
},
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/lib.js",
|
|
12
|
+
"types": "./dist/lib.d.ts"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"files": [
|
|
17
|
+
"dist",
|
|
18
|
+
"LICENSE",
|
|
19
|
+
"README.md"
|
|
20
|
+
],
|
|
21
|
+
"scripts": {
|
|
22
|
+
"build": "tsc",
|
|
23
|
+
"prepublishOnly": "pnpm build",
|
|
24
|
+
"start": "node dist/index.js",
|
|
25
|
+
"dev": "tsx src/index.ts",
|
|
26
|
+
"test": "vitest run",
|
|
27
|
+
"test:watch": "vitest"
|
|
28
|
+
},
|
|
29
|
+
"keywords": [
|
|
30
|
+
"farcaster",
|
|
31
|
+
"lens",
|
|
32
|
+
"nostr",
|
|
33
|
+
"bluesky",
|
|
34
|
+
"atproto",
|
|
35
|
+
"decentralized",
|
|
36
|
+
"social",
|
|
37
|
+
"aggregator",
|
|
38
|
+
"cli",
|
|
39
|
+
"web3"
|
|
40
|
+
],
|
|
41
|
+
"author": "@mattlee.eth",
|
|
42
|
+
"license": "MIT",
|
|
43
|
+
"repository": {
|
|
44
|
+
"type": "git",
|
|
45
|
+
"url": "https://github.com/sphairetra/deso-ag"
|
|
46
|
+
},
|
|
47
|
+
"engines": {
|
|
48
|
+
"node": ">=18"
|
|
49
|
+
},
|
|
50
|
+
"dependencies": {
|
|
51
|
+
"commander": "^12.1.0",
|
|
52
|
+
"nostr-tools": "^2.7.0",
|
|
53
|
+
"websocket-polyfill": "^0.0.3"
|
|
54
|
+
},
|
|
55
|
+
"devDependencies": {
|
|
56
|
+
"@types/node": "^20.11.0",
|
|
57
|
+
"tsx": "^4.7.0",
|
|
58
|
+
"typescript": "^5.3.3",
|
|
59
|
+
"vitest": "^4.0.18"
|
|
60
|
+
},
|
|
61
|
+
"packageManager": "pnpm@10.23.0+sha512.21c4e5698002ade97e4efe8b8b4a89a8de3c85a37919f957e7a0f30f38fbc5bbdd05980ffe29179b2fb6e6e691242e098d945d1601772cad0fef5fb6411e2a4b"
|
|
62
|
+
}
|