mcp-astgl-knowledge 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/knowledge.db +0 -0
- package/dist/discover.d.ts +10 -0
- package/dist/discover.js +147 -0
- package/dist/discover.js.map +1 -0
- package/dist/discovery-db.d.ts +18 -0
- package/dist/discovery-db.js +67 -0
- package/dist/discovery-db.js.map +1 -0
- package/package.json +5 -2
package/data/knowledge.db
CHANGED
|
Binary file
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Content Discovery script.
|
|
4
|
+
* Polls ASTGL site RSS, Substack RSS, and ASTGL sitemap.xml to detect new content.
|
|
5
|
+
* Stores discovered metadata in SQLite with dedup via URL + content hash.
|
|
6
|
+
*
|
|
7
|
+
* Usage: npm run discover
|
|
8
|
+
* Designed to run as OpenClaw cron job (every 6 hours).
|
|
9
|
+
*/
|
|
10
|
+
export {};
|
package/dist/discover.js
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Content Discovery script.
|
|
4
|
+
* Polls ASTGL site RSS, Substack RSS, and ASTGL sitemap.xml to detect new content.
|
|
5
|
+
* Stores discovered metadata in SQLite with dedup via URL + content hash.
|
|
6
|
+
*
|
|
7
|
+
* Usage: npm run discover
|
|
8
|
+
* Designed to run as OpenClaw cron job (every 6 hours).
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from "crypto";
|
|
11
|
+
import Parser from "rss-parser";
|
|
12
|
+
import { initDb, upsertItem, closeDb } from "./discovery-db.js";
|
|
13
|
+
const ASTGL_RSS_URL = process.env.ASTGL_RSS_URL || "https://astgl.ai/rss.xml";
|
|
14
|
+
const SUBSTACK_RSS_URL = process.env.SUBSTACK_RSS_URL || "https://astgl.substack.com/feed";
|
|
15
|
+
const ASTGL_SITEMAP_URL = process.env.ASTGL_SITEMAP_URL || "https://astgl.ai/sitemap-0.xml";
|
|
16
|
+
// WHAT: Generate SHA-256 hash for content dedup
|
|
17
|
+
// WHY: Detects when content at a URL changes (title/description updated)
|
|
18
|
+
function contentHash(title, description, url) {
|
|
19
|
+
const input = `${title || ""}|${description || ""}|${url}`;
|
|
20
|
+
return createHash("sha256").update(input).digest("hex");
|
|
21
|
+
}
|
|
22
|
+
// WHAT: Fetch and parse an RSS feed into DiscoveredItems
|
|
23
|
+
// WHY: rss-parser handles RSS 2.0, Atom, and edge cases cleanly
|
|
24
|
+
async function fetchRss(url, source) {
|
|
25
|
+
const parser = new Parser();
|
|
26
|
+
const feed = await parser.parseURL(url);
|
|
27
|
+
return (feed.items || []).map((item) => {
|
|
28
|
+
const itemUrl = item.link || item.guid || "";
|
|
29
|
+
const title = item.title || null;
|
|
30
|
+
const description = item.contentSnippet || item.content || item.summary || null;
|
|
31
|
+
const pubDate = item.isoDate || item.pubDate || null;
|
|
32
|
+
return {
|
|
33
|
+
url: itemUrl,
|
|
34
|
+
title,
|
|
35
|
+
description: description ? description.slice(0, 500) : null,
|
|
36
|
+
pubDate,
|
|
37
|
+
source,
|
|
38
|
+
contentHash: contentHash(title, description, itemUrl),
|
|
39
|
+
};
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// WHAT: Fetch sitemap.xml and extract URLs
|
|
43
|
+
// WHY: Simple regex is sufficient for well-structured <loc> tags
|
|
44
|
+
async function fetchSitemap(url) {
|
|
45
|
+
const resp = await fetch(url);
|
|
46
|
+
if (!resp.ok) {
|
|
47
|
+
throw new Error(`Sitemap fetch failed: ${resp.status} ${resp.statusText}`);
|
|
48
|
+
}
|
|
49
|
+
const xml = await resp.text();
|
|
50
|
+
const urls = [];
|
|
51
|
+
const locRegex = /<loc>(.*?)<\/loc>/g;
|
|
52
|
+
let match;
|
|
53
|
+
while ((match = locRegex.exec(xml)) !== null) {
|
|
54
|
+
urls.push(match[1]);
|
|
55
|
+
}
|
|
56
|
+
return urls.map((itemUrl) => ({
|
|
57
|
+
url: itemUrl,
|
|
58
|
+
title: null,
|
|
59
|
+
description: null,
|
|
60
|
+
pubDate: null,
|
|
61
|
+
source: "astgl-sitemap",
|
|
62
|
+
contentHash: contentHash(null, null, itemUrl),
|
|
63
|
+
}));
|
|
64
|
+
}
|
|
65
|
+
// WHAT: Upsert a batch of discovered items and tally results
|
|
66
|
+
// WHY: Wraps upsert calls with counting for the summary report
|
|
67
|
+
function processItems(db, items) {
|
|
68
|
+
const result = {
|
|
69
|
+
fetched: items.length,
|
|
70
|
+
new: 0,
|
|
71
|
+
updated: 0,
|
|
72
|
+
skipped: 0,
|
|
73
|
+
errors: 0,
|
|
74
|
+
};
|
|
75
|
+
const upsertAll = db.transaction(() => {
|
|
76
|
+
for (const item of items) {
|
|
77
|
+
if (!item.url) {
|
|
78
|
+
result.errors++;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const status = upsertItem(db, item);
|
|
82
|
+
result[status]++;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
upsertAll();
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
async function main() {
|
|
89
|
+
const db = initDb();
|
|
90
|
+
console.error("Content discovery started");
|
|
91
|
+
const summary = {
|
|
92
|
+
timestamp: new Date().toISOString(),
|
|
93
|
+
sources: {},
|
|
94
|
+
totals: { new: 0, updated: 0, skipped: 0 },
|
|
95
|
+
};
|
|
96
|
+
// WHAT: Define sources to fetch — each source is independent
|
|
97
|
+
// WHY: If one fails, the others still run (cron resilience)
|
|
98
|
+
const sources = [
|
|
99
|
+
{
|
|
100
|
+
name: "astgl-rss",
|
|
101
|
+
fetch: () => fetchRss(ASTGL_RSS_URL, "astgl-rss"),
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
name: "substack-rss",
|
|
105
|
+
fetch: () => fetchRss(SUBSTACK_RSS_URL, "substack-rss"),
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
name: "astgl-sitemap",
|
|
109
|
+
fetch: () => fetchSitemap(ASTGL_SITEMAP_URL),
|
|
110
|
+
},
|
|
111
|
+
];
|
|
112
|
+
for (const source of sources) {
|
|
113
|
+
try {
|
|
114
|
+
console.error(` Fetching ${source.name}...`);
|
|
115
|
+
const items = await source.fetch();
|
|
116
|
+
const result = processItems(db, items);
|
|
117
|
+
summary.sources[source.name] = result;
|
|
118
|
+
console.error(` ${source.name}: ${result.fetched} fetched, ${result.new} new, ${result.updated} updated`);
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
122
|
+
console.error(` ${source.name} failed: ${message}`);
|
|
123
|
+
summary.sources[source.name] = {
|
|
124
|
+
fetched: 0,
|
|
125
|
+
new: 0,
|
|
126
|
+
updated: 0,
|
|
127
|
+
skipped: 0,
|
|
128
|
+
errors: 1,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Compute totals
|
|
133
|
+
for (const result of Object.values(summary.sources)) {
|
|
134
|
+
summary.totals.new += result.new;
|
|
135
|
+
summary.totals.updated += result.updated;
|
|
136
|
+
summary.totals.skipped += result.skipped;
|
|
137
|
+
}
|
|
138
|
+
// JSON summary to stdout for OpenClaw logging
|
|
139
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
140
|
+
closeDb();
|
|
141
|
+
console.error("Content discovery complete");
|
|
142
|
+
}
|
|
143
|
+
main().catch((err) => {
|
|
144
|
+
console.error("Discovery failed:", err);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
});
|
|
147
|
+
//# sourceMappingURL=discover.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discover.js","sourceRoot":"","sources":["../src/discover.ts"],"names":[],"mappings":";AACA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,MAAM,MAAM,YAAY,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAGhE,MAAM,aAAa,GACjB,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,0BAA0B,CAAC;AAC1D,MAAM,gBAAgB,GACpB,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,iCAAiC,CAAC;AACpE,MAAM,iBAAiB,GACrB,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gCAAgC,CAAC;AAgBpE,gDAAgD;AAChD,yEAAyE;AACzE,SAAS,WAAW,CAClB,KAAoB,EACpB,WAA0B,EAC1B,GAAW;IAEX,MAAM,KAAK,GAAG,GAAG,KAAK,IAAI,EAAE,IAAI,WAAW,IAAI,EAAE,IAAI,GAAG,EAAE,CAAC;IAC3D,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,yDAAyD;AACzD,gEAAgE;AAChE,KAAK,UAAU,QAAQ,CACrB,GAAW,EACX,MAAgC;IAEhC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAExC,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;QACjC,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;QAErD,OAAO;YACL,GAAG,EAAE,OAAO;YACZ,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;YAC3D,OAAO;YACP,MAAM;YACN,WAAW,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC;SACtD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,2CAA2C;AAC3C,iEAAiE;AACjE,KAAK,UAAU,YAAY,CAAC,GAAW;IACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,oBAAoB,CAAC;IACtC,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC5B,GAAG,EAAE,OAAO;QACZ,KAAK,EAAE,IAAI;QACX,WAAW,EAAE,IAAI;QACjB,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,eAAwB;QAChC,WAAW,EAAE,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;KAC9C,CAAC,CAAC,CAAC;AACN,CAAC;AAED,6DAA6D;AAC7D,+DAA+D;AAC/D,SAAS,YAAY,CACnB,EAA6B,EAC7B,KAAuB;IAEvB,MAAM,MAAM,GAAiB;QAC3B,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,GAAG,EAAE,CAAC;QACN,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;KACV,CAAC;IAEF,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACd,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAiB,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,SAAS,EAAE,CAAC;IACZ,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAE3C,MAAM,OAAO,GAAqB;QAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;KAC3C,CAAC;IAEF,6DAA6D;IAC7D,4DAA4D;IAC5D,MAAM,OAAO,GAGR;QACH;YACE,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC;SAClD;QACD;YACE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,EAAE,cAAc,CAAC;SACxD;QACD;YACE,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,iBAAiB,CAAC;SAC7C;KACF,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,cAAc,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;YACtC,OAAO,CAAC,KAAK,CACX,KAAK,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,OAAO,aAAa,MAAM,CAAC,GAAG,SAAS,MAAM,CAAC,OAAO,UAAU,CAC5F,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,IAAI,YAAY,OAAO,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG;gBAC7B,OAAO,EAAE,CAAC;gBACV,GAAG,EAAE,CAAC;gBACN,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;aACV,CAAC;QACJ,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;QACzC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;IAC3C,CAAC;IAED,8CAA8C;IAC9C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE9C,OAAO,EAAE,CAAC;IACV,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;AAC9C,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discovery database module.
|
|
3
|
+
* Manages SQLite storage for discovered content metadata.
|
|
4
|
+
* Separate from knowledge.db — discovery.db accumulates over time.
|
|
5
|
+
*/
|
|
6
|
+
import Database from "better-sqlite3";
|
|
7
|
+
export interface DiscoveredItem {
|
|
8
|
+
url: string;
|
|
9
|
+
title: string | null;
|
|
10
|
+
description: string | null;
|
|
11
|
+
pubDate: string | null;
|
|
12
|
+
source: "astgl-rss" | "substack-rss" | "astgl-sitemap";
|
|
13
|
+
contentHash: string;
|
|
14
|
+
}
|
|
15
|
+
export type UpsertResult = "new" | "updated" | "skipped";
|
|
16
|
+
export declare function initDb(): InstanceType<typeof Database>;
|
|
17
|
+
export declare function upsertItem(database: InstanceType<typeof Database>, item: DiscoveredItem): UpsertResult;
|
|
18
|
+
export declare function closeDb(): void;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discovery database module.
|
|
3
|
+
* Manages SQLite storage for discovered content metadata.
|
|
4
|
+
* Separate from knowledge.db — discovery.db accumulates over time.
|
|
5
|
+
*/
|
|
6
|
+
import { join } from "path";
|
|
7
|
+
import { existsSync, mkdirSync } from "fs";
|
|
8
|
+
import Database from "better-sqlite3";
|
|
9
|
+
const DB_PATH = join(import.meta.dirname, "..", "data", "discovery.db");
|
|
10
|
+
let db = null;
|
|
11
|
+
// WHAT: Initialize discovery database with schema
|
|
12
|
+
// WHY: CREATE IF NOT EXISTS preserves accumulated history across runs
|
|
13
|
+
export function initDb() {
|
|
14
|
+
const dataDir = join(import.meta.dirname, "..", "data");
|
|
15
|
+
if (!existsSync(dataDir))
|
|
16
|
+
mkdirSync(dataDir, { recursive: true });
|
|
17
|
+
db = new Database(DB_PATH);
|
|
18
|
+
db.exec(`
|
|
19
|
+
CREATE TABLE IF NOT EXISTS discovered_content (
|
|
20
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
21
|
+
url TEXT NOT NULL UNIQUE,
|
|
22
|
+
title TEXT,
|
|
23
|
+
description TEXT,
|
|
24
|
+
pub_date TEXT,
|
|
25
|
+
source TEXT NOT NULL,
|
|
26
|
+
content_hash TEXT NOT NULL,
|
|
27
|
+
first_seen TEXT NOT NULL,
|
|
28
|
+
last_seen TEXT NOT NULL,
|
|
29
|
+
is_new INTEGER DEFAULT 1
|
|
30
|
+
)
|
|
31
|
+
`);
|
|
32
|
+
return db;
|
|
33
|
+
}
|
|
34
|
+
// WHAT: Insert or update a discovered item based on URL + content hash
|
|
35
|
+
// WHY: Dedup via URL, detect content changes via hash difference
|
|
36
|
+
export function upsertItem(database, item) {
|
|
37
|
+
const now = new Date().toISOString();
|
|
38
|
+
const existing = database
|
|
39
|
+
.prepare("SELECT id, content_hash FROM discovered_content WHERE url = ?")
|
|
40
|
+
.get(item.url);
|
|
41
|
+
if (!existing) {
|
|
42
|
+
database
|
|
43
|
+
.prepare(`INSERT INTO discovered_content (url, title, description, pub_date, source, content_hash, first_seen, last_seen, is_new)
|
|
44
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1)`)
|
|
45
|
+
.run(item.url, item.title, item.description, item.pubDate, item.source, item.contentHash, now, now);
|
|
46
|
+
return "new";
|
|
47
|
+
}
|
|
48
|
+
if (existing.content_hash !== item.contentHash) {
|
|
49
|
+
database
|
|
50
|
+
.prepare(`UPDATE discovered_content
|
|
51
|
+
SET title = ?, description = ?, content_hash = ?, last_seen = ?, is_new = 1
|
|
52
|
+
WHERE id = ?`)
|
|
53
|
+
.run(item.title, item.description, item.contentHash, now, existing.id);
|
|
54
|
+
return "updated";
|
|
55
|
+
}
|
|
56
|
+
database
|
|
57
|
+
.prepare("UPDATE discovered_content SET last_seen = ? WHERE id = ?")
|
|
58
|
+
.run(now, existing.id);
|
|
59
|
+
return "skipped";
|
|
60
|
+
}
|
|
61
|
+
export function closeDb() {
|
|
62
|
+
if (db) {
|
|
63
|
+
db.close();
|
|
64
|
+
db = null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=discovery-db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discovery-db.js","sourceRoot":"","sources":["../src/discovery-db.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC3C,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;AAaxE,IAAI,EAAE,GAAyC,IAAI,CAAC;AAEpD,kDAAkD;AAClD,sEAAsE;AACtE,MAAM,UAAU,MAAM;IACpB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElE,EAAE,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE3B,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;GAaP,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,UAAU,UAAU,CACxB,QAAuC,EACvC,IAAoB;IAEpB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,MAAM,QAAQ,GAAG,QAAQ;SACtB,OAAO,CAAC,+DAA+D,CAAC;SACxE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAqD,CAAC;IAErE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ;aACL,OAAO,CACN;4CACoC,CACrC;aACA,GAAG,CACF,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,GAAG,EACH,GAAG,CACJ,CAAC;QACJ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,QAAQ,CAAC,YAAY,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/C,QAAQ;aACL,OAAO,CACN;;sBAEc,CACf;aACA,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,QAAQ;SACL,OAAO,CAAC,0DAA0D,CAAC;SACnE,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IACzB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,OAAO;IACrB,IAAI,EAAE,EAAE,CAAC;QACP,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,EAAE,GAAG,IAAI,CAAC;IACZ,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-astgl-knowledge",
|
|
3
|
-
"
|
|
3
|
+
"mcpName": "io.github.jmeg8r/astgl-knowledge",
|
|
4
|
+
"version": "1.0.1",
|
|
4
5
|
"description": "MCP server for searching and citing ASTGL articles about MCP servers, local AI, and AI automation",
|
|
5
6
|
"type": "module",
|
|
6
7
|
"bin": {
|
|
@@ -14,13 +15,14 @@
|
|
|
14
15
|
"scripts": {
|
|
15
16
|
"build": "tsc",
|
|
16
17
|
"ingest": "tsx src/ingest.ts",
|
|
18
|
+
"discover": "tsx src/discover.ts",
|
|
17
19
|
"start": "node dist/index.js",
|
|
18
20
|
"dev": "tsx src/index.ts",
|
|
19
21
|
"prepublishOnly": "npm run build"
|
|
20
22
|
},
|
|
21
23
|
"repository": {
|
|
22
24
|
"type": "git",
|
|
23
|
-
"url": "git+https://github.com/
|
|
25
|
+
"url": "git+https://github.com/Jmeg8r/mcp-astgl-knowledge.git"
|
|
24
26
|
},
|
|
25
27
|
"homepage": "https://astgl.ai",
|
|
26
28
|
"keywords": [
|
|
@@ -37,6 +39,7 @@
|
|
|
37
39
|
"dependencies": {
|
|
38
40
|
"@modelcontextprotocol/sdk": "^1.12.1",
|
|
39
41
|
"better-sqlite3": "^11.9.1",
|
|
42
|
+
"rss-parser": "^3.13.0",
|
|
40
43
|
"sqlite-vec": "^0.1.6",
|
|
41
44
|
"zod": "^3.24.4"
|
|
42
45
|
},
|