ahok-skill 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/.prettierrc +8 -0
  2. package/Dockerfile +59 -0
  3. package/RAW_SKILL.md +219 -0
  4. package/README.md +277 -0
  5. package/SKILL.md +58 -0
  6. package/bin/opm.js +268 -0
  7. package/data/openmemory.sqlite +0 -0
  8. package/data/openmemory.sqlite-shm +0 -0
  9. package/data/openmemory.sqlite-wal +0 -0
  10. package/dist/ai/graph.js +293 -0
  11. package/dist/ai/mcp.js +397 -0
  12. package/dist/cli.js +78 -0
  13. package/dist/core/cfg.js +87 -0
  14. package/dist/core/db.js +636 -0
  15. package/dist/core/memory.js +116 -0
  16. package/dist/core/migrate.js +227 -0
  17. package/dist/core/models.js +105 -0
  18. package/dist/core/telemetry.js +57 -0
  19. package/dist/core/types.js +2 -0
  20. package/dist/core/vector/postgres.js +52 -0
  21. package/dist/core/vector/valkey.js +246 -0
  22. package/dist/core/vector_store.js +2 -0
  23. package/dist/index.js +44 -0
  24. package/dist/memory/decay.js +301 -0
  25. package/dist/memory/embed.js +675 -0
  26. package/dist/memory/hsg.js +959 -0
  27. package/dist/memory/reflect.js +131 -0
  28. package/dist/memory/user_summary.js +99 -0
  29. package/dist/migrate.js +9 -0
  30. package/dist/ops/compress.js +255 -0
  31. package/dist/ops/dynamics.js +189 -0
  32. package/dist/ops/extract.js +333 -0
  33. package/dist/ops/ingest.js +214 -0
  34. package/dist/server/index.js +109 -0
  35. package/dist/server/middleware/auth.js +137 -0
  36. package/dist/server/routes/auth.js +186 -0
  37. package/dist/server/routes/compression.js +108 -0
  38. package/dist/server/routes/dashboard.js +399 -0
  39. package/dist/server/routes/docs.js +241 -0
  40. package/dist/server/routes/dynamics.js +312 -0
  41. package/dist/server/routes/ide.js +280 -0
  42. package/dist/server/routes/index.js +33 -0
  43. package/dist/server/routes/keys.js +132 -0
  44. package/dist/server/routes/langgraph.js +61 -0
  45. package/dist/server/routes/memory.js +213 -0
  46. package/dist/server/routes/sources.js +140 -0
  47. package/dist/server/routes/system.js +63 -0
  48. package/dist/server/routes/temporal.js +293 -0
  49. package/dist/server/routes/users.js +101 -0
  50. package/dist/server/routes/vercel.js +57 -0
  51. package/dist/server/server.js +211 -0
  52. package/dist/server.js +3 -0
  53. package/dist/sources/base.js +223 -0
  54. package/dist/sources/github.js +171 -0
  55. package/dist/sources/google_drive.js +166 -0
  56. package/dist/sources/google_sheets.js +112 -0
  57. package/dist/sources/google_slides.js +139 -0
  58. package/dist/sources/index.js +34 -0
  59. package/dist/sources/notion.js +165 -0
  60. package/dist/sources/onedrive.js +143 -0
  61. package/dist/sources/web_crawler.js +166 -0
  62. package/dist/temporal_graph/index.js +20 -0
  63. package/dist/temporal_graph/query.js +240 -0
  64. package/dist/temporal_graph/store.js +116 -0
  65. package/dist/temporal_graph/timeline.js +241 -0
  66. package/dist/temporal_graph/types.js +2 -0
  67. package/dist/utils/chunking.js +60 -0
  68. package/dist/utils/index.js +31 -0
  69. package/dist/utils/keyword.js +94 -0
  70. package/dist/utils/text.js +120 -0
  71. package/nodemon.json +7 -0
  72. package/package.json +50 -0
  73. package/references/api_reference.md +66 -0
  74. package/references/examples.md +45 -0
  75. package/src/ai/graph.ts +363 -0
  76. package/src/ai/mcp.ts +494 -0
  77. package/src/cli.ts +94 -0
  78. package/src/core/cfg.ts +110 -0
  79. package/src/core/db.ts +1052 -0
  80. package/src/core/memory.ts +99 -0
  81. package/src/core/migrate.ts +302 -0
  82. package/src/core/models.ts +107 -0
  83. package/src/core/telemetry.ts +47 -0
  84. package/src/core/types.ts +130 -0
  85. package/src/core/vector/postgres.ts +61 -0
  86. package/src/core/vector/valkey.ts +261 -0
  87. package/src/core/vector_store.ts +9 -0
  88. package/src/index.ts +5 -0
  89. package/src/memory/decay.ts +427 -0
  90. package/src/memory/embed.ts +707 -0
  91. package/src/memory/hsg.ts +1245 -0
  92. package/src/memory/reflect.ts +158 -0
  93. package/src/memory/user_summary.ts +110 -0
  94. package/src/migrate.ts +8 -0
  95. package/src/ops/compress.ts +296 -0
  96. package/src/ops/dynamics.ts +272 -0
  97. package/src/ops/extract.ts +360 -0
  98. package/src/ops/ingest.ts +286 -0
  99. package/src/server/index.ts +159 -0
  100. package/src/server/middleware/auth.ts +156 -0
  101. package/src/server/routes/auth.ts +223 -0
  102. package/src/server/routes/compression.ts +106 -0
  103. package/src/server/routes/dashboard.ts +420 -0
  104. package/src/server/routes/docs.ts +380 -0
  105. package/src/server/routes/dynamics.ts +516 -0
  106. package/src/server/routes/ide.ts +283 -0
  107. package/src/server/routes/index.ts +32 -0
  108. package/src/server/routes/keys.ts +131 -0
  109. package/src/server/routes/langgraph.ts +71 -0
  110. package/src/server/routes/memory.ts +440 -0
  111. package/src/server/routes/sources.ts +111 -0
  112. package/src/server/routes/system.ts +68 -0
  113. package/src/server/routes/temporal.ts +335 -0
  114. package/src/server/routes/users.ts +111 -0
  115. package/src/server/routes/vercel.ts +55 -0
  116. package/src/server/server.js +215 -0
  117. package/src/server.ts +1 -0
  118. package/src/sources/base.ts +257 -0
  119. package/src/sources/github.ts +156 -0
  120. package/src/sources/google_drive.ts +144 -0
  121. package/src/sources/google_sheets.ts +85 -0
  122. package/src/sources/google_slides.ts +115 -0
  123. package/src/sources/index.ts +19 -0
  124. package/src/sources/notion.ts +148 -0
  125. package/src/sources/onedrive.ts +131 -0
  126. package/src/sources/web_crawler.ts +161 -0
  127. package/src/temporal_graph/index.ts +4 -0
  128. package/src/temporal_graph/query.ts +299 -0
  129. package/src/temporal_graph/store.ts +156 -0
  130. package/src/temporal_graph/timeline.ts +319 -0
  131. package/src/temporal_graph/types.ts +41 -0
  132. package/src/utils/chunking.ts +66 -0
  133. package/src/utils/index.ts +25 -0
  134. package/src/utils/keyword.ts +137 -0
  135. package/src/utils/text.ts +115 -0
  136. package/tests/test_api_workspace_management.ts +413 -0
  137. package/tests/test_bulk_delete.ts +267 -0
  138. package/tests/test_omnibus.ts +166 -0
  139. package/tests/test_workspace_management.ts +278 -0
  140. package/tests/verify.ts +104 -0
  141. package/tsconfig.json +15 -0
@@ -0,0 +1,139 @@
1
+ "use strict";
2
+ /**
3
+ * google slides source for openmemory - production grade
4
+ * requires: googleapis
5
+ * env vars: GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON
6
+ */
7
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
8
+ if (k2 === undefined) k2 = k;
9
+ var desc = Object.getOwnPropertyDescriptor(m, k);
10
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
11
+ desc = { enumerable: true, get: function() { return m[k]; } };
12
+ }
13
+ Object.defineProperty(o, k2, desc);
14
+ }) : (function(o, m, k, k2) {
15
+ if (k2 === undefined) k2 = k;
16
+ o[k2] = m[k];
17
+ }));
18
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
19
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
20
+ }) : function(o, v) {
21
+ o["default"] = v;
22
+ });
23
+ var __importStar = (this && this.__importStar) || (function () {
24
+ var ownKeys = function(o) {
25
+ ownKeys = Object.getOwnPropertyNames || function (o) {
26
+ var ar = [];
27
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
28
+ return ar;
29
+ };
30
+ return ownKeys(o);
31
+ };
32
+ return function (mod) {
33
+ if (mod && mod.__esModule) return mod;
34
+ var result = {};
35
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
36
+ __setModuleDefault(result, mod);
37
+ return result;
38
+ };
39
+ })();
40
+ Object.defineProperty(exports, "__esModule", { value: true });
41
+ exports.google_slides_source = void 0;
42
+ const base_1 = require("./base");
43
+ class google_slides_source extends base_1.base_source {
44
+ name = 'google_slides';
45
+ service = null;
46
+ auth = null;
47
+ async _connect(creds) {
48
+ let google;
49
+ try {
50
+ google = await Promise.resolve().then(() => __importStar(require('googleapis'))).then(m => m.google);
51
+ }
52
+ catch {
53
+ throw new base_1.source_config_error('missing deps: npm install googleapis', this.name);
54
+ }
55
+ const scopes = ['https://www.googleapis.com/auth/presentations.readonly'];
56
+ if (creds.credentials_json) {
57
+ this.auth = new google.auth.GoogleAuth({ credentials: creds.credentials_json, scopes });
58
+ }
59
+ else if (creds.service_account_file) {
60
+ this.auth = new google.auth.GoogleAuth({ keyFile: creds.service_account_file, scopes });
61
+ }
62
+ else if (process.env.GOOGLE_CREDENTIALS_JSON) {
63
+ this.auth = new google.auth.GoogleAuth({ credentials: JSON.parse(process.env.GOOGLE_CREDENTIALS_JSON), scopes });
64
+ }
65
+ else if (process.env.GOOGLE_SERVICE_ACCOUNT_FILE) {
66
+ this.auth = new google.auth.GoogleAuth({ keyFile: process.env.GOOGLE_SERVICE_ACCOUNT_FILE, scopes });
67
+ }
68
+ else {
69
+ throw new base_1.source_config_error('no credentials: set GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON', this.name);
70
+ }
71
+ this.service = google.slides({ version: 'v1', auth: this.auth });
72
+ return true;
73
+ }
74
+ async _list_items(filters) {
75
+ if (!filters.presentation_id) {
76
+ throw new base_1.source_config_error('presentation_id is required', this.name);
77
+ }
78
+ const pres = await this.service.presentations.get({ presentationId: filters.presentation_id });
79
+ return (pres.data.slides || []).map((slide, i) => ({
80
+ id: `${filters.presentation_id}#${slide.objectId}`,
81
+ name: `Slide ${i + 1}`,
82
+ type: 'slide',
83
+ index: i,
84
+ presentation_id: filters.presentation_id,
85
+ object_id: slide.objectId
86
+ }));
87
+ }
88
+ async _fetch_item(item_id) {
89
+ const [presentation_id, slide_id] = item_id.includes('#')
90
+ ? item_id.split('#', 2)
91
+ : [item_id, null];
92
+ const pres = await this.service.presentations.get({ presentationId: presentation_id });
93
+ const extract_text = (element) => {
94
+ const texts = [];
95
+ if (element.shape?.text) {
96
+ for (const te of element.shape.text.textElements || []) {
97
+ if (te.textRun)
98
+ texts.push(te.textRun.content || '');
99
+ }
100
+ }
101
+ if (element.table) {
102
+ for (const row of element.table.tableRows || []) {
103
+ for (const cell of row.tableCells || []) {
104
+ if (cell.text) {
105
+ for (const te of cell.text.textElements || []) {
106
+ if (te.textRun)
107
+ texts.push(te.textRun.content || '');
108
+ }
109
+ }
110
+ }
111
+ }
112
+ }
113
+ return texts.join('');
114
+ };
115
+ const all_text = [];
116
+ for (let i = 0; i < (pres.data.slides || []).length; i++) {
117
+ const slide = pres.data.slides[i];
118
+ if (slide_id && slide.objectId !== slide_id)
119
+ continue;
120
+ const slide_texts = [`## Slide ${i + 1}`];
121
+ for (const element of slide.pageElements || []) {
122
+ const txt = extract_text(element);
123
+ if (txt.trim())
124
+ slide_texts.push(txt.trim());
125
+ }
126
+ all_text.push(...slide_texts);
127
+ }
128
+ const text = all_text.join('\n\n');
129
+ return {
130
+ id: item_id,
131
+ name: pres.data.title || 'Untitled Presentation',
132
+ type: 'presentation',
133
+ text,
134
+ data: text,
135
+ meta: { source: 'google_slides', presentation_id, slide_count: pres.data.slides?.length || 0 }
136
+ };
137
+ }
138
+ }
139
+ exports.google_slides_source = google_slides_source;
@@ -0,0 +1,34 @@
1
+ "use strict";
2
+ /**
3
+ * openmemory sources - production-grade data source integrations
4
+ *
5
+ * provides connectors for external data sources:
6
+ * - google drive, sheets, slides
7
+ * - notion
8
+ * - onedrive
9
+ * - github
10
+ * - web crawler
11
+ */
12
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
13
+ if (k2 === undefined) k2 = k;
14
+ var desc = Object.getOwnPropertyDescriptor(m, k);
15
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
16
+ desc = { enumerable: true, get: function() { return m[k]; } };
17
+ }
18
+ Object.defineProperty(o, k2, desc);
19
+ }) : (function(o, m, k, k2) {
20
+ if (k2 === undefined) k2 = k;
21
+ o[k2] = m[k];
22
+ }));
23
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
24
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
25
+ };
26
+ Object.defineProperty(exports, "__esModule", { value: true });
27
+ __exportStar(require("./base"), exports);
28
+ __exportStar(require("./google_drive"), exports);
29
+ __exportStar(require("./google_sheets"), exports);
30
+ __exportStar(require("./google_slides"), exports);
31
+ __exportStar(require("./notion"), exports);
32
+ __exportStar(require("./onedrive"), exports);
33
+ __exportStar(require("./github"), exports);
34
+ __exportStar(require("./web_crawler"), exports);
@@ -0,0 +1,165 @@
1
+ "use strict";
2
+ /**
3
+ * notion source for openmemory - production grade
4
+ * requires: @notionhq/client
5
+ * env vars: NOTION_API_KEY
6
+ */
7
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
8
+ if (k2 === undefined) k2 = k;
9
+ var desc = Object.getOwnPropertyDescriptor(m, k);
10
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
11
+ desc = { enumerable: true, get: function() { return m[k]; } };
12
+ }
13
+ Object.defineProperty(o, k2, desc);
14
+ }) : (function(o, m, k, k2) {
15
+ if (k2 === undefined) k2 = k;
16
+ o[k2] = m[k];
17
+ }));
18
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
19
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
20
+ }) : function(o, v) {
21
+ o["default"] = v;
22
+ });
23
+ var __importStar = (this && this.__importStar) || (function () {
24
+ var ownKeys = function(o) {
25
+ ownKeys = Object.getOwnPropertyNames || function (o) {
26
+ var ar = [];
27
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
28
+ return ar;
29
+ };
30
+ return ownKeys(o);
31
+ };
32
+ return function (mod) {
33
+ if (mod && mod.__esModule) return mod;
34
+ var result = {};
35
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
36
+ __setModuleDefault(result, mod);
37
+ return result;
38
+ };
39
+ })();
40
+ Object.defineProperty(exports, "__esModule", { value: true });
41
+ exports.notion_source = void 0;
42
+ const base_1 = require("./base");
43
+ class notion_source extends base_1.base_source {
44
+ name = 'notion';
45
+ client = null;
46
+ async _connect(creds) {
47
+ let Client;
48
+ try {
49
+ Client = await Promise.resolve().then(() => __importStar(require('@notionhq/client'))).then(m => m.Client);
50
+ }
51
+ catch {
52
+ throw new base_1.source_config_error('missing deps: npm install @notionhq/client', this.name);
53
+ }
54
+ const api_key = creds.api_key || process.env.NOTION_API_KEY;
55
+ if (!api_key) {
56
+ throw new base_1.source_config_error('no credentials: set NOTION_API_KEY', this.name);
57
+ }
58
+ this.client = new Client({ auth: api_key });
59
+ return true;
60
+ }
61
+ extract_title(page) {
62
+ const props = page.properties || {};
63
+ for (const prop of Object.values(props)) {
64
+ if (prop.type === 'title' && prop.title?.[0]) {
65
+ return prop.title[0].plain_text || '';
66
+ }
67
+ }
68
+ return '';
69
+ }
70
+ async _list_items(filters) {
71
+ const results = [];
72
+ if (filters.database_id) {
73
+ let has_more = true;
74
+ let start_cursor;
75
+ while (has_more) {
76
+ const resp = await this.client.databases.query({
77
+ database_id: filters.database_id,
78
+ start_cursor
79
+ });
80
+ for (const page of resp.results) {
81
+ results.push({
82
+ id: page.id,
83
+ name: this.extract_title(page) || 'Untitled',
84
+ type: 'page',
85
+ url: page.url || '',
86
+ last_edited: page.last_edited_time
87
+ });
88
+ }
89
+ has_more = resp.has_more;
90
+ start_cursor = resp.next_cursor;
91
+ }
92
+ }
93
+ else {
94
+ const resp = await this.client.search({ filter: { property: 'object', value: 'page' } });
95
+ for (const page of resp.results) {
96
+ results.push({
97
+ id: page.id,
98
+ name: this.extract_title(page) || 'Untitled',
99
+ type: 'page',
100
+ url: page.url || '',
101
+ last_edited: page.last_edited_time
102
+ });
103
+ }
104
+ }
105
+ return results;
106
+ }
107
+ block_to_text(block) {
108
+ const texts = [];
109
+ const type = block.type;
110
+ const text_blocks = ['paragraph', 'heading_1', 'heading_2', 'heading_3',
111
+ 'bulleted_list_item', 'numbered_list_item', 'quote', 'callout'];
112
+ if (text_blocks.includes(type)) {
113
+ const rich_text = block[type]?.rich_text || [];
114
+ for (const rt of rich_text) {
115
+ texts.push(rt.plain_text || '');
116
+ }
117
+ }
118
+ else if (type === 'code') {
119
+ const rich_text = block.code?.rich_text || [];
120
+ const lang = block.code?.language || '';
121
+ const code = rich_text.map((rt) => rt.plain_text || '').join('');
122
+ texts.push(`\`\`\`${lang}\n${code}\n\`\`\``);
123
+ }
124
+ else if (type === 'to_do') {
125
+ const checked = block.to_do?.checked || false;
126
+ const rich_text = block.to_do?.rich_text || [];
127
+ const prefix = checked ? '[x] ' : '[ ] ';
128
+ texts.push(prefix + rich_text.map((rt) => rt.plain_text || '').join(''));
129
+ }
130
+ return texts.join('');
131
+ }
132
+ async _fetch_item(item_id) {
133
+ const page = await this.client.pages.retrieve({ page_id: item_id });
134
+ const title = this.extract_title(page);
135
+ // get all blocks
136
+ const blocks = [];
137
+ let has_more = true;
138
+ let start_cursor;
139
+ while (has_more) {
140
+ const resp = await this.client.blocks.children.list({
141
+ block_id: item_id,
142
+ start_cursor
143
+ });
144
+ blocks.push(...resp.results);
145
+ has_more = resp.has_more;
146
+ start_cursor = resp.next_cursor;
147
+ }
148
+ const text_parts = title ? [`# ${title}`] : [];
149
+ for (const block of blocks) {
150
+ const txt = this.block_to_text(block);
151
+ if (txt.trim())
152
+ text_parts.push(txt);
153
+ }
154
+ const text = text_parts.join('\n\n');
155
+ return {
156
+ id: item_id,
157
+ name: title || 'Untitled',
158
+ type: 'notion_page',
159
+ text,
160
+ data: text,
161
+ meta: { source: 'notion', page_id: item_id, url: page.url || '', block_count: blocks.length }
162
+ };
163
+ }
164
+ }
165
+ exports.notion_source = notion_source;
@@ -0,0 +1,143 @@
1
+ "use strict";
2
+ /**
3
+ * onedrive source for openmemory - production grade
4
+ * requires: @azure/msal-node
5
+ * env vars: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID
6
+ */
7
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
8
+ if (k2 === undefined) k2 = k;
9
+ var desc = Object.getOwnPropertyDescriptor(m, k);
10
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
11
+ desc = { enumerable: true, get: function() { return m[k]; } };
12
+ }
13
+ Object.defineProperty(o, k2, desc);
14
+ }) : (function(o, m, k, k2) {
15
+ if (k2 === undefined) k2 = k;
16
+ o[k2] = m[k];
17
+ }));
18
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
19
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
20
+ }) : function(o, v) {
21
+ o["default"] = v;
22
+ });
23
+ var __importStar = (this && this.__importStar) || (function () {
24
+ var ownKeys = function(o) {
25
+ ownKeys = Object.getOwnPropertyNames || function (o) {
26
+ var ar = [];
27
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
28
+ return ar;
29
+ };
30
+ return ownKeys(o);
31
+ };
32
+ return function (mod) {
33
+ if (mod && mod.__esModule) return mod;
34
+ var result = {};
35
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
36
+ __setModuleDefault(result, mod);
37
+ return result;
38
+ };
39
+ })();
40
+ Object.defineProperty(exports, "__esModule", { value: true });
41
+ exports.onedrive_source = void 0;
42
+ const base_1 = require("./base");
43
+ class onedrive_source extends base_1.base_source {
44
+ name = 'onedrive';
45
+ access_token = null;
46
+ graph_url = 'https://graph.microsoft.com/v1.0';
47
+ async _connect(creds) {
48
+ if (creds.access_token) {
49
+ this.access_token = creds.access_token;
50
+ return true;
51
+ }
52
+ let msal;
53
+ try {
54
+ msal = await Promise.resolve().then(() => __importStar(require('@azure/msal-node')));
55
+ }
56
+ catch {
57
+ throw new base_1.source_config_error('missing deps: npm install @azure/msal-node', this.name);
58
+ }
59
+ const client_id = creds.client_id || process.env.AZURE_CLIENT_ID;
60
+ const client_secret = creds.client_secret || process.env.AZURE_CLIENT_SECRET;
61
+ const tenant_id = creds.tenant_id || process.env.AZURE_TENANT_ID;
62
+ if (!client_id || !client_secret || !tenant_id) {
63
+ throw new base_1.source_config_error('no credentials: set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID', this.name);
64
+ }
65
+ const app = new msal.ConfidentialClientApplication({
66
+ auth: {
67
+ clientId: client_id,
68
+ clientSecret: client_secret,
69
+ authority: `https://login.microsoftonline.com/${tenant_id}`
70
+ }
71
+ });
72
+ const result = await app.acquireTokenByClientCredential({
73
+ scopes: ['https://graph.microsoft.com/.default']
74
+ });
75
+ if (result?.accessToken) {
76
+ this.access_token = result.accessToken;
77
+ return true;
78
+ }
79
+ throw new base_1.source_auth_error('auth failed: no access token returned', this.name);
80
+ }
81
+ async _list_items(filters) {
82
+ const folder_path = filters.folder_path || '/';
83
+ const user_principal = filters.user_principal;
84
+ const base = user_principal
85
+ ? `${this.graph_url}/users/${user_principal}/drive`
86
+ : `${this.graph_url}/me/drive`;
87
+ const url = folder_path === '/'
88
+ ? `${base}/root/children`
89
+ : `${base}/root:/${folder_path.replace(/^\/|\/$/g, '')}:/children`;
90
+ const results = [];
91
+ let next_url = url;
92
+ while (next_url) {
93
+ const resp = await fetch(next_url, {
94
+ headers: { Authorization: `Bearer ${this.access_token}` }
95
+ });
96
+ if (!resp.ok)
97
+ throw new Error(`http ${resp.status}: ${resp.statusText}`);
98
+ const data = await resp.json();
99
+ for (const item of data.value || []) {
100
+ results.push({
101
+ id: item.id,
102
+ name: item.name,
103
+ type: 'folder' in item ? 'folder' : item.file?.mimeType || 'file',
104
+ size: item.size || 0,
105
+ modified: item.lastModifiedDateTime,
106
+ path: item.parentReference?.path || ''
107
+ });
108
+ }
109
+ next_url = data['@odata.nextLink'] || null;
110
+ }
111
+ return results;
112
+ }
113
+ async _fetch_item(item_id) {
114
+ const base = `${this.graph_url}/me/drive`;
115
+ const meta_resp = await fetch(`${base}/items/${item_id}`, {
116
+ headers: { Authorization: `Bearer ${this.access_token}` }
117
+ });
118
+ if (!meta_resp.ok)
119
+ throw new Error(`http ${meta_resp.status}`);
120
+ const meta = await meta_resp.json();
121
+ const content_resp = await fetch(`${base}/items/${item_id}/content`, {
122
+ headers: { Authorization: `Bearer ${this.access_token}` },
123
+ redirect: 'follow'
124
+ });
125
+ if (!content_resp.ok)
126
+ throw new Error(`http ${content_resp.status}`);
127
+ const data = Buffer.from(await content_resp.arrayBuffer());
128
+ let text = '';
129
+ try {
130
+ text = data.toString('utf-8');
131
+ }
132
+ catch { }
133
+ return {
134
+ id: item_id,
135
+ name: meta.name || 'unknown',
136
+ type: meta.file?.mimeType || 'unknown',
137
+ text,
138
+ data,
139
+ meta: { source: 'onedrive', item_id, size: meta.size || 0, mime_type: meta.file?.mimeType || '' }
140
+ };
141
+ }
142
+ }
143
+ exports.onedrive_source = onedrive_source;
@@ -0,0 +1,166 @@
1
+ "use strict";
2
+ /**
3
+ * web crawler source for openmemory - production grade
4
+ * requires: cheerio (for html parsing)
5
+ * no auth required for public urls
6
+ */
7
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
8
+ if (k2 === undefined) k2 = k;
9
+ var desc = Object.getOwnPropertyDescriptor(m, k);
10
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
11
+ desc = { enumerable: true, get: function() { return m[k]; } };
12
+ }
13
+ Object.defineProperty(o, k2, desc);
14
+ }) : (function(o, m, k, k2) {
15
+ if (k2 === undefined) k2 = k;
16
+ o[k2] = m[k];
17
+ }));
18
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
19
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
20
+ }) : function(o, v) {
21
+ o["default"] = v;
22
+ });
23
+ var __importStar = (this && this.__importStar) || (function () {
24
+ var ownKeys = function(o) {
25
+ ownKeys = Object.getOwnPropertyNames || function (o) {
26
+ var ar = [];
27
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
28
+ return ar;
29
+ };
30
+ return ownKeys(o);
31
+ };
32
+ return function (mod) {
33
+ if (mod && mod.__esModule) return mod;
34
+ var result = {};
35
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
36
+ __setModuleDefault(result, mod);
37
+ return result;
38
+ };
39
+ })();
40
+ Object.defineProperty(exports, "__esModule", { value: true });
41
+ exports.web_crawler_source = void 0;
42
+ const base_1 = require("./base");
43
+ class web_crawler_source extends base_1.base_source {
44
+ name = 'web_crawler';
45
+ max_pages;
46
+ max_depth;
47
+ timeout;
48
+ visited = new Set();
49
+ crawled = [];
50
+ constructor(user_id, config) {
51
+ super(user_id, config);
52
+ this.max_pages = config?.max_pages || 50;
53
+ this.max_depth = config?.max_depth || 3;
54
+ this.timeout = config?.timeout || 30000;
55
+ }
56
+ async _connect() {
57
+ return true; // no auth needed
58
+ }
59
+ async _list_items(filters) {
60
+ if (!filters.start_url) {
61
+ throw new base_1.source_config_error('start_url is required', this.name);
62
+ }
63
+ let cheerio;
64
+ try {
65
+ cheerio = await Promise.resolve().then(() => __importStar(require('cheerio')));
66
+ }
67
+ catch {
68
+ throw new base_1.source_config_error('missing deps: npm install cheerio', this.name);
69
+ }
70
+ this.visited.clear();
71
+ this.crawled = [];
72
+ const base_url = new URL(filters.start_url);
73
+ const base_domain = base_url.hostname;
74
+ const to_visit = [{ url: filters.start_url, depth: 0 }];
75
+ const follow_links = filters.follow_links !== false;
76
+ while (to_visit.length > 0 && this.crawled.length < this.max_pages) {
77
+ const { url, depth } = to_visit.shift();
78
+ if (this.visited.has(url) || depth > this.max_depth)
79
+ continue;
80
+ this.visited.add(url);
81
+ try {
82
+ const controller = new AbortController();
83
+ const timeout_id = setTimeout(() => controller.abort(), this.timeout);
84
+ const resp = await fetch(url, {
85
+ headers: { 'User-Agent': 'OpenMemory-Crawler/1.0 (compatible)' },
86
+ signal: controller.signal
87
+ });
88
+ clearTimeout(timeout_id);
89
+ if (!resp.ok)
90
+ continue;
91
+ const content_type = resp.headers.get('content-type') || '';
92
+ if (!content_type.includes('text/html'))
93
+ continue;
94
+ const html = await resp.text();
95
+ const $ = cheerio.load(html);
96
+ const title = $('title').text() || url;
97
+ this.crawled.push({
98
+ id: url,
99
+ name: title.trim(),
100
+ type: 'webpage',
101
+ url,
102
+ depth
103
+ });
104
+ // find and queue links
105
+ if (follow_links && depth < this.max_depth) {
106
+ $('a[href]').each((_, el) => {
107
+ try {
108
+ const href = $(el).attr('href');
109
+ if (!href)
110
+ return;
111
+ const full_url = new URL(href, url);
112
+ if (full_url.hostname !== base_domain)
113
+ return;
114
+ const clean_url = `${full_url.protocol}//${full_url.hostname}${full_url.pathname}`;
115
+ if (!this.visited.has(clean_url)) {
116
+ to_visit.push({ url: clean_url, depth: depth + 1 });
117
+ }
118
+ }
119
+ catch { }
120
+ });
121
+ }
122
+ }
123
+ catch (e) {
124
+ console.warn(`[web_crawler] failed to fetch ${url}: ${e.message}`);
125
+ }
126
+ }
127
+ return this.crawled;
128
+ }
129
+ async _fetch_item(item_id) {
130
+ let cheerio;
131
+ try {
132
+ cheerio = await Promise.resolve().then(() => __importStar(require('cheerio')));
133
+ }
134
+ catch {
135
+ throw new base_1.source_config_error('missing deps: npm install cheerio', this.name);
136
+ }
137
+ const controller = new AbortController();
138
+ const timeout_id = setTimeout(() => controller.abort(), this.timeout);
139
+ const resp = await fetch(item_id, {
140
+ headers: { 'User-Agent': 'OpenMemory-Crawler/1.0 (compatible)' },
141
+ signal: controller.signal
142
+ });
143
+ clearTimeout(timeout_id);
144
+ if (!resp.ok)
145
+ throw new Error(`http ${resp.status}: ${resp.statusText}`);
146
+ const html = await resp.text();
147
+ const $ = cheerio.load(html);
148
+ // remove noise
149
+ $('script, style, nav, footer, header, aside').remove();
150
+ const title = $('title').text() || item_id;
151
+ // get main content
152
+ const main = $('main').length ? $('main') : $('article').length ? $('article') : $('body');
153
+ let text = main.text();
154
+ // clean up whitespace
155
+ text = text.split('\n').map((l) => l.trim()).filter(Boolean).join('\n');
156
+ return {
157
+ id: item_id,
158
+ name: title.trim(),
159
+ type: 'webpage',
160
+ text,
161
+ data: text,
162
+ meta: { source: 'web_crawler', url: item_id, char_count: text.length }
163
+ };
164
+ }
165
+ }
166
+ exports.web_crawler_source = web_crawler_source;
@@ -0,0 +1,20 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./types"), exports);
18
+ __exportStar(require("./store"), exports);
19
+ __exportStar(require("./query"), exports);
20
+ __exportStar(require("./timeline"), exports);