@kaelio/ktx 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/assets/python/{kaelio_ktx-0.13.0-py3-none-any.whl → kaelio_ktx-0.14.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/cli-program.js +1 -1
  5. package/dist/commands/ingest-commands.d.ts +9 -0
  6. package/dist/commands/ingest-commands.js +37 -1
  7. package/dist/commands/knowledge-commands.js +3 -0
  8. package/dist/commands/setup-commands.js +14 -1
  9. package/dist/connection-drivers.d.ts +2 -0
  10. package/dist/connection-drivers.js +7 -3
  11. package/dist/connection.d.ts +3 -0
  12. package/dist/connection.js +27 -0
  13. package/dist/connectors/bigquery/connector.d.ts +17 -6
  14. package/dist/connectors/bigquery/connector.js +152 -76
  15. package/dist/connectors/bigquery/dialect.d.ts +2 -2
  16. package/dist/connectors/clickhouse/connector.d.ts +1 -0
  17. package/dist/connectors/clickhouse/connector.js +45 -16
  18. package/dist/connectors/clickhouse/dialect.d.ts +2 -2
  19. package/dist/connectors/mongodb/connector.d.ts +69 -0
  20. package/dist/connectors/mongodb/connector.js +307 -0
  21. package/dist/connectors/mongodb/dialect.d.ts +17 -0
  22. package/dist/connectors/mongodb/dialect.js +50 -0
  23. package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
  24. package/dist/connectors/mongodb/live-database-introspection.js +24 -0
  25. package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
  26. package/dist/connectors/mongodb/schema-inference.js +110 -0
  27. package/dist/connectors/mysql/connector.d.ts +1 -0
  28. package/dist/connectors/mysql/connector.js +18 -2
  29. package/dist/connectors/mysql/dialect.d.ts +2 -2
  30. package/dist/connectors/postgres/connector.d.ts +1 -0
  31. package/dist/connectors/postgres/connector.js +20 -3
  32. package/dist/connectors/postgres/dialect.d.ts +2 -2
  33. package/dist/connectors/snowflake/connector.d.ts +1 -0
  34. package/dist/connectors/snowflake/connector.js +27 -3
  35. package/dist/connectors/snowflake/dialect.d.ts +2 -2
  36. package/dist/connectors/sqlite/connector.d.ts +11 -1
  37. package/dist/connectors/sqlite/connector.js +120 -17
  38. package/dist/connectors/sqlite/dialect.d.ts +2 -2
  39. package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
  40. package/dist/connectors/sqlite/read-query-child.js +23 -0
  41. package/dist/connectors/sqlserver/connector.d.ts +2 -0
  42. package/dist/connectors/sqlserver/connector.js +23 -5
  43. package/dist/connectors/sqlserver/dialect.d.ts +2 -2
  44. package/dist/context/cache/content-result-cache.d.ts +36 -0
  45. package/dist/context/cache/content-result-cache.js +14 -0
  46. package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
  47. package/dist/context/cache/sqlite-content-result-cache.js +223 -0
  48. package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
  49. package/dist/context/connections/bigquery-identifiers.js +7 -0
  50. package/dist/context/connections/configured-connections.d.ts +9 -0
  51. package/dist/context/connections/configured-connections.js +18 -0
  52. package/dist/context/connections/dialects.d.ts +30 -4
  53. package/dist/context/connections/dialects.js +38 -11
  54. package/dist/context/connections/drivers.js +21 -0
  55. package/dist/context/connections/gdrive-config.d.ts +19 -0
  56. package/dist/context/connections/gdrive-config.js +57 -0
  57. package/dist/context/connections/query-deadline.d.ts +31 -0
  58. package/dist/context/connections/query-deadline.js +37 -0
  59. package/dist/context/connections/read-only-sql.d.ts +5 -0
  60. package/dist/context/connections/read-only-sql.js +139 -1
  61. package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
  62. package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
  63. package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
  64. package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
  65. package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
  66. package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
  67. package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
  68. package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
  69. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
  70. package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
  71. package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
  72. package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
  73. package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
  74. package/dist/context/ingest/adapters/gdrive/types.js +36 -0
  75. package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
  76. package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
  77. package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
  78. package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
  79. package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
  80. package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
  81. package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
  82. package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
  83. package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
  84. package/dist/context/ingest/adapters/live-database/stage.js +5 -5
  85. package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
  86. package/dist/context/ingest/adapters/metabase/types.js +1 -1
  87. package/dist/context/ingest/artifact-gates.d.ts +32 -1
  88. package/dist/context/ingest/artifact-gates.js +85 -18
  89. package/dist/context/ingest/final-gate-prune.d.ts +35 -0
  90. package/dist/context/ingest/final-gate-prune.js +229 -0
  91. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  92. package/dist/context/ingest/ingest-bundle.runner.js +459 -240
  93. package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
  94. package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
  95. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  96. package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
  97. package/dist/context/ingest/local-adapters.js +6 -0
  98. package/dist/context/ingest/local-bundle-runtime.js +7 -2
  99. package/dist/context/ingest/local-ingest.js +1 -1
  100. package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
  101. package/dist/context/ingest/local-stage-ingest.js +3 -1
  102. package/dist/context/ingest/ports.d.ts +3 -0
  103. package/dist/context/ingest/report-snapshot.js +13 -3
  104. package/dist/context/ingest/reports.d.ts +3 -3
  105. package/dist/context/ingest/reports.js +3 -1
  106. package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
  107. package/dist/context/ingest/stages/build-wu-context.js +2 -1
  108. package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
  109. package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
  110. package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
  111. package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
  112. package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
  113. package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
  114. package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
  115. package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
  116. package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
  117. package/dist/context/ingest/types.d.ts +3 -0
  118. package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
  119. package/dist/context/ingest/wiki-body-refs.js +37 -8
  120. package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
  121. package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
  122. package/dist/context/ingest/work-unit-cache.d.ts +67 -0
  123. package/dist/context/ingest/work-unit-cache.js +230 -0
  124. package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
  125. package/dist/context/llm/ai-sdk-runtime.js +5 -0
  126. package/dist/context/llm/claude-code-runtime.d.ts +4 -1
  127. package/dist/context/llm/claude-code-runtime.js +38 -8
  128. package/dist/context/llm/codex-runtime.d.ts +4 -1
  129. package/dist/context/llm/codex-runtime.js +48 -35
  130. package/dist/context/llm/runtime-port.d.ts +26 -0
  131. package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
  132. package/dist/context/llm/subprocess-generate-object-child.js +34 -0
  133. package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
  134. package/dist/context/llm/subprocess-generate-object.js +122 -0
  135. package/dist/context/mcp/context-tools.d.ts +2 -0
  136. package/dist/context/mcp/context-tools.js +111 -17
  137. package/dist/context/mcp/local-project-ports.d.ts +7 -0
  138. package/dist/context/mcp/local-project-ports.js +29 -0
  139. package/dist/context/mcp/logger.d.ts +24 -0
  140. package/dist/context/mcp/logger.js +49 -0
  141. package/dist/context/mcp/server.js +2 -0
  142. package/dist/context/mcp/types.d.ts +17 -0
  143. package/dist/context/memory/memory-agent.service.js +1 -1
  144. package/dist/context/project/config.d.ts +42 -0
  145. package/dist/context/project/config.js +5 -0
  146. package/dist/context/project/driver-schemas.d.ts +20 -0
  147. package/dist/context/project/driver-schemas.js +50 -1
  148. package/dist/context/project/project.js +1 -1
  149. package/dist/context/project/setup-config.js +1 -0
  150. package/dist/context/scan/description-generation.d.ts +4 -0
  151. package/dist/context/scan/description-generation.js +100 -13
  152. package/dist/context/scan/enabled-tables.d.ts +5 -0
  153. package/dist/context/scan/enabled-tables.js +13 -1
  154. package/dist/context/scan/enrichment-state.d.ts +50 -7
  155. package/dist/context/scan/enrichment-state.js +30 -13
  156. package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
  157. package/dist/context/scan/local-enrichment-artifacts.js +155 -32
  158. package/dist/context/scan/local-enrichment.d.ts +39 -4
  159. package/dist/context/scan/local-enrichment.js +345 -92
  160. package/dist/context/scan/local-scan.d.ts +4 -1
  161. package/dist/context/scan/local-scan.js +54 -13
  162. package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
  163. package/dist/context/scan/local-structural-artifacts.js +5 -0
  164. package/dist/context/scan/object-introspection.d.ts +21 -0
  165. package/dist/context/scan/object-introspection.js +35 -0
  166. package/dist/context/scan/relationship-benchmarks.js +5 -3
  167. package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
  168. package/dist/context/scan/relationship-composite-candidates.js +13 -1
  169. package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
  170. package/dist/context/scan/relationship-detection-budget.js +53 -0
  171. package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
  172. package/dist/context/scan/relationship-diagnostics.js +2 -0
  173. package/dist/context/scan/relationship-discovery.d.ts +9 -3
  174. package/dist/context/scan/relationship-discovery.js +27 -4
  175. package/dist/context/scan/relationship-llm-proposal.js +36 -27
  176. package/dist/context/scan/relationship-profiling.d.ts +7 -3
  177. package/dist/context/scan/relationship-profiling.js +53 -41
  178. package/dist/context/scan/relationship-validation.d.ts +6 -4
  179. package/dist/context/scan/relationship-validation.js +46 -32
  180. package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
  181. package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
  182. package/dist/context/scan/types.d.ts +5 -3
  183. package/dist/context/scan/types.js +2 -0
  184. package/dist/context/sl/local-query.js +5 -0
  185. package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
  186. package/dist/context/sl/semantic-layer.service.js +1 -1
  187. package/dist/context/sl/source-files.d.ts +5 -0
  188. package/dist/context/sl/source-files.js +17 -11
  189. package/dist/context/sl/tools/connection-id-schema.js +1 -1
  190. package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
  191. package/dist/context/sql-analysis/dialect-notes.js +40 -0
  192. package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
  193. package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
  194. package/dist/context/sql-analysis/dialects/mysql.md +9 -0
  195. package/dist/context/sql-analysis/dialects/postgres.md +10 -0
  196. package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
  197. package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
  198. package/dist/context/sql-analysis/dialects/tsql.md +10 -0
  199. package/dist/context/wiki/keys.js +1 -1
  200. package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
  201. package/dist/context/wiki/knowledge-wiki.service.js +2 -1
  202. package/dist/context/wiki/local-knowledge.d.ts +13 -0
  203. package/dist/context/wiki/local-knowledge.js +50 -6
  204. package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
  205. package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
  206. package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
  207. package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
  208. package/dist/context/wiki/types.d.ts +6 -0
  209. package/dist/context-build-view.d.ts +3 -0
  210. package/dist/context-build-view.js +1 -0
  211. package/dist/knowledge.d.ts +2 -0
  212. package/dist/knowledge.js +12 -1
  213. package/dist/local-adapters.js +11 -0
  214. package/dist/mcp-http-server.js +15 -1
  215. package/dist/mcp-server-factory.d.ts +2 -0
  216. package/dist/mcp-server-factory.js +15 -1
  217. package/dist/mcp-stdio-server.js +10 -2
  218. package/dist/notion-page-picker.js +1 -1
  219. package/dist/prompts/memory_agent_external_ingest.md +2 -0
  220. package/dist/public-ingest.d.ts +3 -1
  221. package/dist/public-ingest.js +3 -0
  222. package/dist/scan.d.ts +3 -1
  223. package/dist/scan.js +7 -0
  224. package/dist/setup-databases.d.ts +1 -1
  225. package/dist/setup-databases.js +43 -1
  226. package/dist/setup-sources.d.ts +5 -1
  227. package/dist/setup-sources.js +124 -48
  228. package/dist/setup.d.ts +3 -0
  229. package/dist/setup.js +6 -1
  230. package/dist/skills/analytics/SKILL.md +200 -2
  231. package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
  232. package/dist/skills/wiki_capture/SKILL.md +24 -0
  233. package/dist/sql.js +4 -0
  234. package/dist/status-project.d.ts +4 -0
  235. package/dist/status-project.js +54 -2
  236. package/dist/telemetry/events.d.ts +2 -2
  237. package/dist/text-ingest.d.ts +4 -0
  238. package/dist/text-ingest.js +58 -29
  239. package/dist/verbatim-ingest.d.ts +46 -0
  240. package/dist/verbatim-ingest.js +193 -0
  241. package/package.json +5 -1
  242. package/dist/context/ingest/final-gate-repair.d.ts +0 -28
  243. package/dist/context/ingest/final-gate-repair.js +0 -91
@@ -91,6 +91,143 @@ export function assertReadOnlySql(sql) {
91
91
  assertSingleSqlStatement(trimmed);
92
92
  return trimmed;
93
93
  }
94
+ function isSqlIdentifierPart(char) {
95
+ return char !== undefined && /[A-Za-z0-9_$]/.test(char);
96
+ }
97
+ function keywordAt(sql, index, keyword) {
98
+ if (sql.slice(index, index + keyword.length).toLowerCase() !== keyword.toLowerCase()) {
99
+ return false;
100
+ }
101
+ return !isSqlIdentifierPart(sql[index - 1]) && !isSqlIdentifierPart(sql[index + keyword.length]);
102
+ }
103
+ function skipWhitespaceAndComments(sql, index) {
104
+ let current = index;
105
+ while (current < sql.length) {
106
+ while (/\s/.test(sql[current] ?? '')) {
107
+ current += 1;
108
+ }
109
+ if (sql.startsWith('--', current) || sql.startsWith('/*', current)) {
110
+ current = skipQuotedOrComment(sql, current);
111
+ continue;
112
+ }
113
+ return current;
114
+ }
115
+ return current;
116
+ }
117
+ function skipBracketIdentifier(sql, index) {
118
+ let current = index + 1;
119
+ while (current < sql.length) {
120
+ if (sql[current] === ']') {
121
+ if (sql[current + 1] === ']') {
122
+ current += 2;
123
+ continue;
124
+ }
125
+ return current + 1;
126
+ }
127
+ current += 1;
128
+ }
129
+ return -1;
130
+ }
131
+ function skipBacktickIdentifier(sql, index) {
132
+ let current = index + 1;
133
+ while (current < sql.length) {
134
+ if (sql[current] === '`') {
135
+ if (sql[current + 1] === '`') {
136
+ current += 2;
137
+ continue;
138
+ }
139
+ return current + 1;
140
+ }
141
+ current += 1;
142
+ }
143
+ return -1;
144
+ }
145
+ function skipIdentifier(sql, index) {
146
+ if (sql[index] === '"') {
147
+ const skipped = skipQuotedOrComment(sql, index);
148
+ return skipped > index ? skipped : -1;
149
+ }
150
+ if (sql[index] === '[') {
151
+ return skipBracketIdentifier(sql, index);
152
+ }
153
+ if (sql[index] === '`') {
154
+ return skipBacktickIdentifier(sql, index);
155
+ }
156
+ let current = index;
157
+ while (isSqlIdentifierPart(sql[current])) {
158
+ current += 1;
159
+ }
160
+ return current > index ? current : -1;
161
+ }
162
+ function skipBalancedParentheses(sql, index) {
163
+ if (sql[index] !== '(') {
164
+ return -1;
165
+ }
166
+ let current = index;
167
+ let depth = 0;
168
+ while (current < sql.length) {
169
+ const skipped = skipQuotedOrComment(sql, current);
170
+ if (skipped > current) {
171
+ current = skipped;
172
+ continue;
173
+ }
174
+ if (sql[current] === '(') {
175
+ depth += 1;
176
+ }
177
+ else if (sql[current] === ')') {
178
+ depth -= 1;
179
+ if (depth === 0) {
180
+ return current + 1;
181
+ }
182
+ }
183
+ current += 1;
184
+ }
185
+ return -1;
186
+ }
187
+ /** @internal */
188
+ export function hoistLeadingCte(sql) {
189
+ const trimmed = sql.trim();
190
+ if (!keywordAt(trimmed, 0, 'with')) {
191
+ return { withPrefix: '', body: sql };
192
+ }
193
+ let current = skipWhitespaceAndComments(trimmed, 4);
194
+ if (keywordAt(trimmed, current, 'recursive')) {
195
+ current = skipWhitespaceAndComments(trimmed, current + 'recursive'.length);
196
+ }
197
+ while (current < trimmed.length) {
198
+ current = skipIdentifier(trimmed, current);
199
+ if (current < 0) {
200
+ return { withPrefix: '', body: trimmed };
201
+ }
202
+ current = skipWhitespaceAndComments(trimmed, current);
203
+ if (trimmed[current] === '(') {
204
+ current = skipBalancedParentheses(trimmed, current);
205
+ if (current < 0) {
206
+ return { withPrefix: '', body: trimmed };
207
+ }
208
+ current = skipWhitespaceAndComments(trimmed, current);
209
+ }
210
+ if (!keywordAt(trimmed, current, 'as')) {
211
+ return { withPrefix: '', body: trimmed };
212
+ }
213
+ current = skipWhitespaceAndComments(trimmed, current + 2);
214
+ current = skipBalancedParentheses(trimmed, current);
215
+ if (current < 0) {
216
+ return { withPrefix: '', body: trimmed };
217
+ }
218
+ current = skipWhitespaceAndComments(trimmed, current);
219
+ if (trimmed[current] === ',') {
220
+ current = skipWhitespaceAndComments(trimmed, current + 1);
221
+ continue;
222
+ }
223
+ const body = trimmed.slice(current).trimStart();
224
+ if (!body) {
225
+ return { withPrefix: '', body: trimmed };
226
+ }
227
+ return { withPrefix: `${trimmed.slice(0, current).trimEnd()} `, body };
228
+ }
229
+ return { withPrefix: '', body: trimmed };
230
+ }
94
231
  // `assertReadOnlySql` deliberately keeps trailing semicolons, comments, and
95
232
  // whitespace (e.g. `select 1; -- done`) — harmless for direct single-statement
96
233
  // execution. A row-limit subquery wrapper needs a bare expression instead: a
@@ -130,5 +267,6 @@ export function limitSqlForExecution(sql, maxRows) {
130
267
  if (!Number.isInteger(maxRows) || maxRows <= 0) {
131
268
  throw new KtxQueryError('maxRows must be a positive integer.');
132
269
  }
133
- return `select * from (${trimmed}) as ktx_query_result limit ${maxRows}`;
270
+ const { withPrefix, body } = hoistLeadingCte(trimmed);
271
+ return `${withPrefix}select * from (${body}) as ktx_query_result limit ${maxRows}`;
134
272
  }
@@ -0,0 +1,3 @@
1
+ import type { ChunkResult, DiffSet, ScopeDescriptor } from '../../types.js';
2
+ export declare function chunkGdriveStagedDir(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
3
+ export declare function describeGdriveScope(stagedDir: string): Promise<ScopeDescriptor>;
@@ -0,0 +1,74 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { readdir, readFile } from 'node:fs/promises';
3
+ import { join, relative } from 'node:path';
4
+ import { gdriveManifestSchema, gdriveMetadataSchema } from './types.js';
5
+ const GDRIVE_RECONCILE_GUIDANCE = 'Synthesize durable wiki knowledge from this Google Doc. Preserve product definitions, process documentation, and operating rules as wiki pages. Do not create semantic-layer sources from gdrive content in v1.';
6
+ function normalizeRawPath(path) {
7
+ return path.replace(/\\/g, '/');
8
+ }
9
+ async function walk(root) {
10
+ const entries = await readdir(root, { withFileTypes: true, recursive: true });
11
+ return entries
12
+ .filter((entry) => entry.isFile())
13
+ .map((entry) => normalizeRawPath(relative(root, join(entry.parentPath, entry.name))))
14
+ .sort();
15
+ }
16
+ function safeUnitKey(path) {
17
+ return `gdrive-${path.replace(/^docs\//, '').replace(/\/page\.md$/, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '')}`;
18
+ }
19
+ async function readManifest(stagedDir) {
20
+ try {
21
+ return gdriveManifestSchema.parse(JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8')));
22
+ }
23
+ catch (error) {
24
+ throw new Error(`Invalid gdrive manifest: ${error instanceof Error ? error.message : String(error)}`);
25
+ }
26
+ }
27
+ export async function chunkGdriveStagedDir(stagedDir, diffSet) {
28
+ const files = await walk(stagedDir);
29
+ const manifest = await readManifest(stagedDir);
30
+ const touched = diffSet
31
+ ? new Set([...diffSet.added, ...diffSet.modified].map((path) => normalizeRawPath(path)))
32
+ : null;
33
+ const workUnits = [];
34
+ for (const pagePath of files.filter((path) => path.endsWith('/page.md'))) {
35
+ const metadataPath = pagePath.replace(/\/page\.md$/, '/metadata.json');
36
+ const primary = [metadataPath, pagePath].filter((path) => files.includes(path));
37
+ if (touched && !primary.some((path) => touched.has(path))) {
38
+ continue;
39
+ }
40
+ const metadata = gdriveMetadataSchema.parse(JSON.parse(await readFile(join(stagedDir, metadataPath), 'utf-8')));
41
+ const rawFiles = touched ? primary.filter((path) => touched.has(path)).sort() : primary.sort();
42
+ const dependencyPaths = ['manifest.json'].filter((path) => !rawFiles.includes(path));
43
+ const excluded = new Set([...rawFiles, ...dependencyPaths]);
44
+ const peerFileIndex = files.filter((path) => !excluded.has(path)).sort();
45
+ workUnits.push({
46
+ unitKey: safeUnitKey(pagePath),
47
+ displayLabel: metadata.path,
48
+ rawFiles,
49
+ dependencyPaths,
50
+ peerFileIndex,
51
+ notes: GDRIVE_RECONCILE_GUIDANCE,
52
+ });
53
+ }
54
+ return {
55
+ workUnits,
56
+ eviction: diffSet && diffSet.deleted.length > 0
57
+ ? { deletedRawPaths: diffSet.deleted.map((path) => normalizeRawPath(path)).sort() }
58
+ : undefined,
59
+ reconcileNotes: ['Google Drive docs are knowledge-only in v1; keep output in wiki pages unless later follow-up work expands scope.'],
60
+ contextReport: { capped: false, warnings: manifest.warnings },
61
+ };
62
+ }
63
+ export async function describeGdriveScope(stagedDir) {
64
+ const manifest = await readManifest(stagedDir);
65
+ const scopeKey = JSON.stringify({
66
+ folderId: manifest.folderId,
67
+ recursive: manifest.recursive,
68
+ });
69
+ const fingerprint = createHash('sha256').update(scopeKey).digest('hex');
70
+ return {
71
+ fingerprint,
72
+ isPathInScope: (rawPath) => rawPath === 'manifest.json' || rawPath.startsWith('docs/'),
73
+ };
74
+ }
@@ -0,0 +1 @@
1
+ export declare function detectGdriveStagedDir(stagedDir: string): Promise<boolean>;
@@ -0,0 +1,20 @@
1
+ import { readFile, readdir } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ export async function detectGdriveStagedDir(stagedDir) {
4
+ try {
5
+ const manifest = JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8'));
6
+ if (manifest.source === 'gdrive') {
7
+ return true;
8
+ }
9
+ }
10
+ catch {
11
+ // Fall through to structural detection.
12
+ }
13
+ try {
14
+ const entries = await readdir(stagedDir, { withFileTypes: true, recursive: true });
15
+ return entries.some((entry) => entry.isFile() && entry.name === 'page.md');
16
+ }
17
+ catch {
18
+ return false;
19
+ }
20
+ }
@@ -0,0 +1,6 @@
1
+ import type { GdriveManifest, GdrivePullConfig } from './types.js';
2
+ export declare function fetchGdriveSnapshot(params: {
3
+ key: unknown;
4
+ config: GdrivePullConfig;
5
+ stagedDir: string;
6
+ }): Promise<GdriveManifest>;
@@ -0,0 +1,95 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { mkdir, writeFile } from 'node:fs/promises';
3
+ import { dirname, join } from 'node:path';
4
+ import { createGoogleDocsClients, driveFolderChildrenQuery } from './gdrive-client.js';
5
+ import { normalizeGoogleDocToMarkdown } from './normalize.js';
6
+ import { GDRIVE_DOC_MIME_TYPE, GDRIVE_FOLDER_MIME_TYPE, GDRIVE_SOURCE_KEY } from './types.js';
7
+ async function writeJson(path, value) {
8
+ await mkdir(dirname(path), { recursive: true });
9
+ await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, 'utf-8');
10
+ }
11
+ async function writeText(path, value) {
12
+ await mkdir(dirname(path), { recursive: true });
13
+ await writeFile(path, value.endsWith('\n') ? value : `${value}\n`, 'utf-8');
14
+ }
15
+ function slugifySegment(value) {
16
+ const normalized = value
17
+ .normalize('NFKD')
18
+ .replace(/[^\x00-\x7F]/g, '')
19
+ .replace(/[^a-zA-Z0-9]+/g, '-')
20
+ .replace(/^-+|-+$/g, '')
21
+ .toLowerCase();
22
+ return normalized || 'untitled';
23
+ }
24
+ function compactSegment(value, maxLength = 24) {
25
+ const slug = slugifySegment(value);
26
+ return slug.length > maxLength ? slug.slice(0, maxLength).replace(/-+$/g, '') || 'untitled' : slug;
27
+ }
28
+ function shortHash(value, length = 10) {
29
+ return createHash('sha1').update(value).digest('hex').slice(0, length);
30
+ }
31
+ function gdriveDocDirName(title, fileId) {
32
+ return `${compactSegment(title)}-${shortHash(fileId)}`;
33
+ }
34
+ async function listFolderFiles(drive, folderId, recursive, parents = []) {
35
+ const q = driveFolderChildrenQuery(folderId);
36
+ const docs = [];
37
+ const skipped = [];
38
+ let pageToken;
39
+ do {
40
+ const page = await drive.listFiles({ q, pageToken });
41
+ for (const file of page.files) {
42
+ if (file.mimeType === GDRIVE_FOLDER_MIME_TYPE) {
43
+ if (recursive) {
44
+ const nested = await listFolderFiles(drive, file.id, true, [...parents, file.name]);
45
+ docs.push(...nested.docs);
46
+ skipped.push(...nested.skipped);
47
+ }
48
+ continue;
49
+ }
50
+ if (file.mimeType !== GDRIVE_DOC_MIME_TYPE) {
51
+ skipped.push({ externalId: file.id, reason: `unsupported mime type: ${file.mimeType}` });
52
+ continue;
53
+ }
54
+ docs.push({ file, drivePath: parents, folderId });
55
+ }
56
+ pageToken = page.nextPageToken ?? undefined;
57
+ } while (pageToken);
58
+ return { docs, skipped };
59
+ }
60
+ export async function fetchGdriveSnapshot(params) {
61
+ await mkdir(params.stagedDir, { recursive: true });
62
+ const clients = createGoogleDocsClients(params.key);
63
+ const { docs, skipped } = await listFolderFiles(clients.drive, params.config.folderId, params.config.recursive);
64
+ for (const { file, drivePath, folderId } of docs) {
65
+ const document = await clients.docs.getDocument(file.id);
66
+ const title = (document.title?.trim() || file.name).trim();
67
+ const relDir = join('docs', ...drivePath.map((segment) => compactSegment(segment)), gdriveDocDirName(title, file.id));
68
+ const markdownBody = normalizeGoogleDocToMarkdown(document);
69
+ const pageMarkdown = [`# ${title}`, markdownBody].filter(Boolean).join('\n\n');
70
+ await writeJson(join(params.stagedDir, relDir, 'metadata.json'), {
71
+ id: file.id,
72
+ title,
73
+ path: [...drivePath, title].join(' / ') || title,
74
+ url: file.webViewLink,
75
+ mimeType: file.mimeType,
76
+ folderId,
77
+ drivePath,
78
+ modifiedTime: file.modifiedTime,
79
+ });
80
+ await writeText(join(params.stagedDir, relDir, 'page.md'), pageMarkdown);
81
+ }
82
+ const manifest = {
83
+ source: GDRIVE_SOURCE_KEY,
84
+ folderId: params.config.folderId,
85
+ recursive: params.config.recursive,
86
+ fetchedAt: new Date().toISOString(),
87
+ fileCount: docs.length,
88
+ skipped,
89
+ warnings: skipped.length > 0
90
+ ? [`Skipped ${skipped.length} non-Google-Doc file(s); only Google Docs are ingested in v1.`]
91
+ : [],
92
+ };
93
+ await writeJson(join(params.stagedDir, 'manifest.json'), manifest);
94
+ return manifest;
95
+ }
@@ -0,0 +1,30 @@
1
+ import type { GdriveFileRecord, GoogleDocsDocument } from './types.js';
2
+ export interface GoogleDriveClient {
3
+ listFiles(args: {
4
+ q: string;
5
+ pageToken?: string;
6
+ }): Promise<{
7
+ files: GdriveFileRecord[];
8
+ nextPageToken: string | null;
9
+ }>;
10
+ getFile(fileId: string): Promise<GdriveFileRecord | null>;
11
+ }
12
+ export interface GoogleDocsClients {
13
+ drive: GoogleDriveClient;
14
+ docs: {
15
+ getDocument(documentId: string): Promise<GoogleDocsDocument>;
16
+ };
17
+ }
18
+ /** @internal Retries transient Google API responses (429/5xx) honoring Retry-After. */
19
+ export declare function fetchWithGoogleRetry(doFetch: () => Promise<Response>, options?: {
20
+ maxAttempts?: number;
21
+ sleep?: (ms: number) => Promise<void>;
22
+ }): Promise<Response>;
23
+ /** Builds the Drive query for the non-trashed direct children of a folder, escaping the folder id. */
24
+ export declare function driveFolderChildrenQuery(folderId: string): string;
25
+ /**
26
+ * Confirms `folderId` resolves to a folder the service account can read, then counts the
27
+ * Google Docs directly inside it. Throws a caller-facing error when the id is missing or not a folder.
28
+ */
29
+ export declare function verifyGdriveFolderAndCountDocs(drive: GoogleDriveClient, folderId: string): Promise<number>;
30
+ export declare function createGoogleDocsClients(rawKey: unknown): GoogleDocsClients;
@@ -0,0 +1,132 @@
1
+ import { JWT } from 'google-auth-library';
2
+ import { GDRIVE_DOC_MIME_TYPE, GDRIVE_FOLDER_MIME_TYPE, GDRIVE_SCOPES, gdriveServiceAccountKeySchema } from './types.js';
3
+ const GOOGLE_DRIVE_BASE_URL = 'https://www.googleapis.com/drive/v3';
4
+ const GOOGLE_DOCS_BASE_URL = 'https://docs.googleapis.com/v1';
5
+ const GOOGLE_FILE_FIELDS = 'id,name,mimeType,parents,webViewLink,modifiedTime';
6
+ const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
7
+ const MAX_REQUEST_ATTEMPTS = 4;
8
+ function defaultSleep(ms) {
9
+ return new Promise((resolve) => setTimeout(resolve, ms));
10
+ }
11
+ function retryDelayMs(attempt, retryAfterHeader) {
12
+ const retryAfterSeconds = retryAfterHeader ? Number.parseInt(retryAfterHeader, 10) : Number.NaN;
13
+ if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds >= 0) {
14
+ return Math.min(retryAfterSeconds * 1000, 30_000);
15
+ }
16
+ return Math.min(500 * 2 ** attempt, 8_000);
17
+ }
18
+ /** @internal Retries transient Google API responses (429/5xx) honoring Retry-After. */
19
+ export async function fetchWithGoogleRetry(doFetch, options = {}) {
20
+ const maxAttempts = options.maxAttempts ?? MAX_REQUEST_ATTEMPTS;
21
+ const sleep = options.sleep ?? defaultSleep;
22
+ let response = await doFetch();
23
+ for (let attempt = 1; attempt < maxAttempts && !response.ok && RETRYABLE_STATUSES.has(response.status); attempt += 1) {
24
+ await sleep(retryDelayMs(attempt - 1, response.headers.get('retry-after')));
25
+ response = await doFetch();
26
+ }
27
+ return response;
28
+ }
29
+ async function parseGoogleResponse(response) {
30
+ if (!response.ok) {
31
+ const body = await response.text();
32
+ throw new Error(`Google API request failed (${response.status}): ${body || response.statusText}`);
33
+ }
34
+ return (await response.json());
35
+ }
36
+ async function authorizedFetch(client, url) {
37
+ return fetchWithGoogleRetry(async () => {
38
+ const headers = await client.getRequestHeaders(url);
39
+ return fetch(url, { headers });
40
+ });
41
+ }
42
+ function isGoogleApiFileRecord(file) {
43
+ return typeof file.id === 'string' && typeof file.name === 'string' && typeof file.mimeType === 'string';
44
+ }
45
+ function toFileRecord(file) {
46
+ return {
47
+ id: file.id,
48
+ name: file.name,
49
+ mimeType: file.mimeType,
50
+ parents: Array.isArray(file.parents) ? file.parents.filter((parent) => typeof parent === 'string') : [],
51
+ webViewLink: typeof file.webViewLink === 'string' ? file.webViewLink : null,
52
+ modifiedTime: typeof file.modifiedTime === 'string' ? file.modifiedTime : null,
53
+ };
54
+ }
55
+ function escapeDriveQueryValue(value) {
56
+ return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
57
+ }
58
+ /** Builds the Drive query for the non-trashed direct children of a folder, escaping the folder id. */
59
+ export function driveFolderChildrenQuery(folderId) {
60
+ return `'${escapeDriveQueryValue(folderId)}' in parents and trashed = false`;
61
+ }
62
+ /**
63
+ * Confirms `folderId` resolves to a folder the service account can read, then counts the
64
+ * Google Docs directly inside it. Throws a caller-facing error when the id is missing or not a folder.
65
+ */
66
+ export async function verifyGdriveFolderAndCountDocs(drive, folderId) {
67
+ const folder = await drive.getFile(folderId);
68
+ if (!folder) {
69
+ throw new Error(`Google Drive folder "${folderId}" is not accessible. Share it with the service account email and verify folder_id.`);
70
+ }
71
+ if (folder.mimeType !== GDRIVE_FOLDER_MIME_TYPE) {
72
+ throw new Error(`Google Drive id "${folderId}" is not a folder (mimeType: ${folder.mimeType}).`);
73
+ }
74
+ const q = driveFolderChildrenQuery(folderId);
75
+ let docs = 0;
76
+ let pageToken;
77
+ do {
78
+ const page = await drive.listFiles({ q, pageToken });
79
+ docs += page.files.filter((file) => file.mimeType === GDRIVE_DOC_MIME_TYPE).length;
80
+ pageToken = page.nextPageToken ?? undefined;
81
+ } while (pageToken);
82
+ return docs;
83
+ }
84
+ export function createGoogleDocsClients(rawKey) {
85
+ const key = gdriveServiceAccountKeySchema.parse(rawKey);
86
+ const client = new JWT({
87
+ email: key.client_email,
88
+ key: key.private_key,
89
+ scopes: [...GDRIVE_SCOPES],
90
+ });
91
+ return {
92
+ drive: {
93
+ async listFiles(args) {
94
+ const params = new URLSearchParams({
95
+ q: args.q,
96
+ supportsAllDrives: 'true',
97
+ includeItemsFromAllDrives: 'true',
98
+ pageSize: '1000',
99
+ fields: `nextPageToken,files(${GOOGLE_FILE_FIELDS})`,
100
+ });
101
+ if (args.pageToken) {
102
+ params.set('pageToken', args.pageToken);
103
+ }
104
+ const response = await authorizedFetch(client, `${GOOGLE_DRIVE_BASE_URL}/files?${params.toString()}`);
105
+ const parsed = await parseGoogleResponse(response);
106
+ return {
107
+ files: (parsed.files ?? []).filter(isGoogleApiFileRecord).map(toFileRecord),
108
+ nextPageToken: typeof parsed.nextPageToken === 'string' ? parsed.nextPageToken : null,
109
+ };
110
+ },
111
+ async getFile(fileId) {
112
+ const params = new URLSearchParams({ supportsAllDrives: 'true', fields: GOOGLE_FILE_FIELDS });
113
+ const response = await authorizedFetch(client, `${GOOGLE_DRIVE_BASE_URL}/files/${encodeURIComponent(fileId)}?${params.toString()}`);
114
+ if (response.status === 404) {
115
+ return null;
116
+ }
117
+ const file = await parseGoogleResponse(response);
118
+ return isGoogleApiFileRecord(file) ? toFileRecord(file) : null;
119
+ },
120
+ },
121
+ docs: {
122
+ async getDocument(documentId) {
123
+ const params = new URLSearchParams({
124
+ includeTabsContent: 'true',
125
+ suggestionsViewMode: 'PREVIEW_WITHOUT_SUGGESTIONS',
126
+ });
127
+ const response = await authorizedFetch(client, `${GOOGLE_DOCS_BASE_URL}/documents/${documentId}?${params.toString()}`);
128
+ return await parseGoogleResponse(response);
129
+ },
130
+ },
131
+ };
132
+ }
@@ -0,0 +1,11 @@
1
+ import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js';
2
+ export declare class GdriveSourceAdapter implements SourceAdapter {
3
+ readonly source = "gdrive";
4
+ readonly skillNames: string[];
5
+ readonly reconcileSkillNames: string[];
6
+ readonly evidenceIndexing: "documents";
7
+ detect(stagedDir: string): Promise<boolean>;
8
+ fetch(pullConfig: unknown, stagedDir: string, _ctx: FetchContext): Promise<void>;
9
+ chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
10
+ describeScope(stagedDir: string): Promise<ScopeDescriptor>;
11
+ }
@@ -0,0 +1,27 @@
1
+ import { chunkGdriveStagedDir, describeGdriveScope } from './chunk.js';
2
+ import { detectGdriveStagedDir } from './detect.js';
3
+ import { fetchGdriveSnapshot } from './fetch.js';
4
+ import { gdrivePullConfigSchema } from './types.js';
5
+ export class GdriveSourceAdapter {
6
+ source = 'gdrive';
7
+ skillNames = ['gdrive_synthesize'];
8
+ reconcileSkillNames = [];
9
+ evidenceIndexing = 'documents';
10
+ detect(stagedDir) {
11
+ return detectGdriveStagedDir(stagedDir);
12
+ }
13
+ async fetch(pullConfig, stagedDir, _ctx) {
14
+ const config = gdrivePullConfigSchema.parse(pullConfig);
15
+ await fetchGdriveSnapshot({
16
+ key: JSON.parse(config.serviceAccountKey),
17
+ config,
18
+ stagedDir,
19
+ });
20
+ }
21
+ chunk(stagedDir, diffSet) {
22
+ return chunkGdriveStagedDir(stagedDir, diffSet);
23
+ }
24
+ describeScope(stagedDir) {
25
+ return describeGdriveScope(stagedDir);
26
+ }
27
+ }
@@ -0,0 +1,2 @@
1
+ import type { GoogleDocsDocument } from './types.js';
2
+ export declare function normalizeGoogleDocToMarkdown(document: GoogleDocsDocument): string;