@surveilr/bootstrap-sql 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,844 @@
1
+ import {
2
+ codeNotebookCell,
3
+ codeNotebookState,
4
+ device,
5
+ orchestrationNature,
6
+ orchestrationSession,
7
+ orchestrationSessionEntry,
8
+ orchestrationSessionExec,
9
+ orchestrationSessionIssue,
10
+ orchestrationSessionLog,
11
+ orchestrationSessionState,
12
+ uniformResource,
13
+ uniformResourceEdge,
14
+ uniformResourceGraph,
15
+ urIngestSession,
16
+ urIngestSessionFsPath,
17
+ urIngestSessionFsPathEntry,
18
+ urIngestSessionImapAccount,
19
+ urIngestSessionImapAcctFolder,
20
+ urIngestSessionImapAcctFolderMessage,
21
+ urIngestSessionTask
22
+ } from "./chunk-HZLAQ5XF.js";
23
+
24
+ // views.ts
25
+ import { sqliteView } from "drizzle-orm/sqlite-core";
26
+ import { sql, count, min, max, sum, eq, and } from "drizzle-orm";
27
+ var codeNotebookCellVersions = sqliteView("code_notebook_cell_versions").as((qb) => {
28
+ return qb.select({
29
+ notebookName: codeNotebookCell.notebookName,
30
+ notebookKernelId: codeNotebookCell.notebookKernelId,
31
+ cellName: codeNotebookCell.cellName,
32
+ versions: sql`COUNT(*) OVER(PARTITION BY ${codeNotebookCell.notebookName}, ${codeNotebookCell.cellName})`.as("versions"),
33
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId
34
+ }).from(codeNotebookCell).orderBy(codeNotebookCell.notebookName, codeNotebookCell.cellName);
35
+ });
36
+ var codeNotebookCellLatest = sqliteView("code_notebook_cell_latest").as((qb) => {
37
+ return qb.select({
38
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
39
+ notebookKernelId: codeNotebookCell.notebookKernelId,
40
+ notebookName: codeNotebookCell.notebookName,
41
+ cellName: codeNotebookCell.cellName,
42
+ interpretableCode: codeNotebookCell.interpretableCode,
43
+ interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
44
+ description: codeNotebookCell.description,
45
+ cellGovernance: codeNotebookCell.cellGovernance,
46
+ arguments: codeNotebookCell.arguments,
47
+ activityLog: codeNotebookCell.activityLog,
48
+ versionTimestamp: sql`COALESCE(${codeNotebookCell.updatedAt}, ${codeNotebookCell.createdAt})`.as("version_timestamp")
49
+ }).from(codeNotebookCell).where(sql`${codeNotebookCell.codeNotebookCellId} IN (
50
+ SELECT code_notebook_cell_id
51
+ FROM (
52
+ SELECT code_notebook_cell_id,
53
+ ROW_NUMBER() OVER (
54
+ PARTITION BY code_notebook_cell_id
55
+ ORDER BY COALESCE(updated_at, created_at) DESC
56
+ ) AS rn
57
+ FROM code_notebook_cell
58
+ ) ranked WHERE rn = 1
59
+ )`);
60
+ });
61
+ var codeNotebookSqlCellMigratableVersion = sqliteView("code_notebook_sql_cell_migratable_version").as((qb) => {
62
+ return qb.select({
63
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
64
+ notebookName: codeNotebookCell.notebookName,
65
+ cellName: codeNotebookCell.cellName,
66
+ interpretableCode: codeNotebookCell.interpretableCode,
67
+ interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
68
+ isIdempotent: sql`CASE WHEN ${codeNotebookCell.cellName} LIKE '%_once_%' THEN FALSE ELSE TRUE END`.as("is_idempotent"),
69
+ versionTimestamp: sql`COALESCE(${codeNotebookCell.updatedAt}, ${codeNotebookCell.createdAt})`.as("version_timestamp")
70
+ }).from(codeNotebookCell).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
71
+ });
72
+ var codeNotebookSqlCellMigratable = sqliteView("code_notebook_sql_cell_migratable").as((qb) => {
73
+ return qb.select({
74
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
75
+ notebookKernelId: codeNotebookCell.notebookKernelId,
76
+ notebookName: codeNotebookCell.notebookName,
77
+ cellName: codeNotebookCell.cellName,
78
+ interpretableCode: codeNotebookCell.interpretableCode,
79
+ interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
80
+ description: codeNotebookCell.description,
81
+ cellGovernance: codeNotebookCell.cellGovernance,
82
+ arguments: codeNotebookCell.arguments,
83
+ activityLog: codeNotebookCell.activityLog,
84
+ isIdempotent: sql`CASE WHEN ${codeNotebookCell.cellName} LIKE '%_once_%' THEN FALSE ELSE TRUE END`.as("is_idempotent")
85
+ }).from(codeNotebookCell).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
86
+ });
87
+ var codeNotebookSqlCellMigratableState = sqliteView("code_notebook_sql_cell_migratable_state").as((qb) => {
88
+ return qb.select({
89
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
90
+ notebookKernelId: codeNotebookCell.notebookKernelId,
91
+ notebookName: codeNotebookCell.notebookName,
92
+ cellName: codeNotebookCell.cellName,
93
+ interpretableCode: codeNotebookCell.interpretableCode,
94
+ interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
95
+ description: codeNotebookCell.description,
96
+ cellGovernance: codeNotebookCell.cellGovernance,
97
+ arguments: codeNotebookCell.arguments,
98
+ activityLog: codeNotebookCell.activityLog,
99
+ fromState: codeNotebookState.fromState,
100
+ toState: codeNotebookState.toState,
101
+ transitionReason: codeNotebookState.transitionReason,
102
+ transitionResult: codeNotebookState.transitionResult,
103
+ transitionedAt: codeNotebookState.transitionedAt
104
+ }).from(codeNotebookCell).innerJoin(codeNotebookState, eq(codeNotebookCell.codeNotebookCellId, codeNotebookState.codeNotebookCellId)).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
105
+ });
106
+ var codeNotebookSqlCellMigratableNotExecuted = sqliteView("code_notebook_sql_cell_migratable_not_executed").as((qb) => {
107
+ return qb.select({
108
+ codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
109
+ notebookKernelId: codeNotebookCell.notebookKernelId,
110
+ notebookName: codeNotebookCell.notebookName,
111
+ cellName: codeNotebookCell.cellName,
112
+ interpretableCode: codeNotebookCell.interpretableCode,
113
+ interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
114
+ description: codeNotebookCell.description,
115
+ cellGovernance: codeNotebookCell.cellGovernance,
116
+ arguments: codeNotebookCell.arguments,
117
+ activityLog: codeNotebookCell.activityLog
118
+ }).from(codeNotebookCell).leftJoin(codeNotebookState, and(
119
+ eq(codeNotebookCell.codeNotebookCellId, codeNotebookState.codeNotebookCellId),
120
+ eq(codeNotebookState.toState, "EXECUTED")
121
+ )).where(
122
+ and(
123
+ eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook"),
124
+ sql`${codeNotebookState.codeNotebookCellId} IS NULL`
125
+ )
126
+ ).orderBy(codeNotebookCell.cellName);
127
+ });
128
+ var codeNotebookMigrationSql = sqliteView("code_notebook_migration_sql", {}).as(sql`
129
+ SELECT 'BEGIN TRANSACTION;
130
+
131
+ ' ||
132
+ 'CREATE TEMP TABLE IF NOT EXISTS "session_state_ephemeral" (
133
+ "key" TEXT PRIMARY KEY NOT NULL,
134
+ "value" TEXT NOT NULL
135
+ );
136
+
137
+ ' ||
138
+ GROUP_CONCAT(
139
+ CASE
140
+ -- Case 1: Non-idempotent and already executed
141
+ WHEN c.is_idempotent = FALSE AND s.code_notebook_cell_id IS NOT NULL THEN
142
+ '-- ' || c.notebook_name || '.' || c.cell_name || ' not included because it is non-idempotent and was already executed on ' || s.transitioned_at || '
143
+
144
+ '
145
+ -- Case 2: Idempotent and not yet executed, idempotent and being reapplied, or non-idempotent and being run for the first time
146
+ ELSE
147
+ '-- ' || c.notebook_name || '.' || c.cell_name || '
148
+ ' ||
149
+ CASE
150
+ -- First execution (non-idempotent or idempotent)
151
+ WHEN s.code_notebook_cell_id IS NULL THEN
152
+ '-- Executing for the first time.
153
+ '
154
+ -- Reapplying execution (idempotent)
155
+ ELSE
156
+ '-- Reapplying execution. Last executed on ' || s.transitioned_at || '
157
+ '
158
+ END ||
159
+ c.interpretable_code || '
160
+ ' ||
161
+ 'INSERT INTO code_notebook_state (code_notebook_state_id, code_notebook_cell_id, from_state, to_state, transition_reason, created_at) ' ||
162
+ 'VALUES (' ||
163
+ '''' || c.code_notebook_cell_id || '__' || strftime('%Y%m%d%H%M%S', 'now') || '''' || ', ' ||
164
+ '''' || c.code_notebook_cell_id || '''' || ', ' ||
165
+ '''MIGRATION_CANDIDATE''' || ', ' ||
166
+ '''EXECUTED''' || ', ' ||
167
+ CASE
168
+ WHEN s.code_notebook_cell_id IS NULL THEN '''Migration'''
169
+ ELSE '''Reapplication'''
170
+ END || ', ' ||
171
+ 'CURRENT_TIMESTAMP' || ')' || '
172
+ ' ||
173
+ 'ON CONFLICT(code_notebook_cell_id, from_state, to_state) DO UPDATE SET updated_at = CURRENT_TIMESTAMP, ' ||
174
+ 'transition_reason = ''Reapplied ' || datetime('now', 'localtime') || ''';' || '
175
+ '
176
+ END,
177
+ '
178
+ '
179
+ ) || '
180
+
181
+ COMMIT;' as migration_sql
182
+ FROM code_notebook_sql_cell_migratable c
183
+ LEFT JOIN code_notebook_state s
184
+ ON c.code_notebook_cell_id = s.code_notebook_cell_id AND s.to_state = 'EXECUTED'
185
+ ORDER BY c.cell_name`);
186
+ var imapGraph = sqliteView("imap_graph").as((qb) => {
187
+ return qb.select({
188
+ graphName: uniformResourceEdge.graphName,
189
+ uniformResourceId: uniformResource.uniformResourceId,
190
+ nature: uniformResource.nature,
191
+ uri: uniformResource.uri,
192
+ content: uniformResource.content,
193
+ urIngestSessionImapAcctFolderMessageId: urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId,
194
+ ingestImapAcctFolderId: urIngestSessionImapAcctFolderMessage.ingestImapAcctFolderId,
195
+ messageId: urIngestSessionImapAcctFolderMessage.messageId
196
+ }).from(uniformResourceEdge).innerJoin(uniformResource, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(urIngestSessionImapAcctFolderMessage, eq(uniformResourceEdge.nodeId, urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId)).where(eq(uniformResourceEdge.graphName, "imap"));
197
+ });
198
+ var filesystemGraph = sqliteView("filesystem_graph").as((qb) => {
199
+ return qb.select({
200
+ graphName: uniformResourceEdge.graphName,
201
+ uniformResourceId: uniformResource.uniformResourceId,
202
+ nature: uniformResource.nature,
203
+ uri: uniformResource.uri,
204
+ urIngestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
205
+ rootPath: urIngestSessionFsPath.rootPath
206
+ }).from(uniformResourceEdge).innerJoin(uniformResource, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(urIngestSessionFsPath, eq(uniformResourceEdge.nodeId, urIngestSessionFsPath.urIngestSessionFsPathId)).where(eq(uniformResourceEdge.graphName, "filesystem"));
207
+ });
208
+ var consoleInformationSchemaTable = sqliteView("console_information_schema_table", {}).as(sql`
209
+ SELECT
210
+ tbl.name AS table_name,
211
+ col.name AS column_name,
212
+ col.type AS data_type,
213
+ CASE WHEN col.pk = 1 THEN 'Yes' ELSE 'No' END AS is_primary_key,
214
+ CASE WHEN col."notnull" = 1 THEN 'Yes' ELSE 'No' END AS is_not_null,
215
+ col.dflt_value AS default_value,
216
+ 'console/info-schema/table.sql?name=' || tbl.name || '&stats=yes' as info_schema_web_ui_path,
217
+ '[Content](console/info-schema/table.sql?name=' || tbl.name || '&stats=yes)' as info_schema_link_abbrev_md,
218
+ '[' || tbl.name || ' (table) Schema](console/info-schema/table.sql?name=' || tbl.name || '&stats=yes)' as info_schema_link_full_md,
219
+ 'console/content/table/' || tbl.name || '.sql?stats=yes' as content_web_ui_path,
220
+ '[Content]($SITE_PREFIX_URL/console/content/table/' || tbl.name || '.sql?stats=yes)' as content_web_ui_link_abbrev_md,
221
+ '[' || tbl.name || ' (table) Content](console/content/table/' || tbl.name || '.sql?stats=yes)' as content_web_ui_link_full_md,
222
+ tbl.sql as sql_ddl
223
+ FROM sqlite_master tbl
224
+ JOIN pragma_table_info(tbl.name) col
225
+ WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
226
+ `);
227
+ var consoleInformationSchemaView = sqliteView("console_information_schema_view", {}).as(sql`
228
+ SELECT
229
+ vw.name AS view_name,
230
+ col.name AS column_name,
231
+ col.type AS data_type,
232
+ '/console/info-schema/view.sql?name=' || vw.name || '&stats=yes' as info_schema_web_ui_path,
233
+ '[Content](console/info-schema/view.sql?name=' || vw.name || '&stats=yes)' as info_schema_link_abbrev_md,
234
+ '[' || vw.name || ' (view) Schema](console/info-schema/view.sql?name=' || vw.name || '&stats=yes)' as info_schema_link_full_md,
235
+ '/console/content/view/' || vw.name || '.sql?stats=yes' as content_web_ui_path,
236
+ '[Content]($SITE_PREFIX_URL/console/content/view/' || vw.name || '.sql?stats=yes)' as content_web_ui_link_abbrev_md,
237
+ '[' || vw.name || ' (view) Content](console/content/view/' || vw.name || '.sql?stats=yes)' as content_web_ui_link_full_md,
238
+ vw.sql as sql_ddl
239
+ FROM sqlite_master vw
240
+ JOIN pragma_table_info(vw.name) col
241
+ WHERE vw.type = 'view' AND vw.name NOT LIKE 'sqlite_%'
242
+ `);
243
+ var consoleContentTabular = sqliteView("console_content_tabular").as((qb) => {
244
+ const tableQuery = qb.select({
245
+ tabularNature: sql`'table'`.as("tabular_nature"),
246
+ tabularName: sql`table_name`.as("tabular_name"),
247
+ infoSchemaWebUiPath: sql`info_schema_web_ui_path`.as("info_schema_web_ui_path"),
248
+ infoSchemaLinkAbbrevMd: sql`info_schema_link_abbrev_md`.as("info_schema_link_abbrev_md"),
249
+ infoSchemaLinkFullMd: sql`info_schema_link_full_md`.as("info_schema_link_full_md"),
250
+ contentWebUiPath: sql`content_web_ui_path`.as("content_web_ui_path"),
251
+ contentWebUiLinkAbbrevMd: sql`content_web_ui_link_abbrev_md`.as("content_web_ui_link_abbrev_md"),
252
+ contentWebUiLinkFullMd: sql`content_web_ui_link_full_md`.as("content_web_ui_link_full_md")
253
+ }).from(consoleInformationSchemaTable);
254
+ const viewQuery = qb.select({
255
+ tabularNature: sql`'view'`.as("tabular_nature"),
256
+ tabularName: sql`view_name`.as("tabular_name"),
257
+ infoSchemaWebUiPath: sql`info_schema_web_ui_path`.as("info_schema_web_ui_path"),
258
+ infoSchemaLinkAbbrevMd: sql`info_schema_link_abbrev_md`.as("info_schema_link_abbrev_md"),
259
+ infoSchemaLinkFullMd: sql`info_schema_link_full_md`.as("info_schema_link_full_md"),
260
+ contentWebUiPath: sql`content_web_ui_path`.as("content_web_ui_path"),
261
+ contentWebUiLinkAbbrevMd: sql`content_web_ui_link_abbrev_md`.as("content_web_ui_link_abbrev_md"),
262
+ contentWebUiLinkFullMd: sql`content_web_ui_link_full_md`.as("content_web_ui_link_full_md")
263
+ }).from(consoleInformationSchemaView);
264
+ return tableQuery.unionAll(viewQuery);
265
+ });
266
+ var consoleInformationSchemaTableColFkey = sqliteView("console_information_schema_table_col_fkey", {}).as(sql`
267
+ SELECT
268
+ tbl.name AS table_name,
269
+ f."from" AS column_name,
270
+ f."from" || ' references ' || f."table" || '.' || f."to" AS foreign_key
271
+ FROM sqlite_master tbl
272
+ JOIN pragma_foreign_key_list(tbl.name) f
273
+ WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
274
+ `);
275
+ var consoleInformationSchemaTableColIndex = sqliteView("console_information_schema_table_col_index", {}).as(sql`
276
+ SELECT
277
+ tbl.name AS table_name,
278
+ pi.name AS column_name,
279
+ idx.name AS index_name
280
+ FROM sqlite_master tbl
281
+ JOIN pragma_index_list(tbl.name) idx
282
+ JOIN pragma_index_info(idx.name) pi
283
+ WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
284
+ `);
285
+ var rssdStatisticsOverview = sqliteView("rssd_statistics_overview").as((qb) => {
286
+ return qb.select({
287
+ dbSizeMb: sql`(SELECT ROUND(page_count * page_size / (1024.0 * 1024), 2) FROM pragma_page_count(), pragma_page_size())`.as("db_size_mb"),
288
+ dbSizeGb: sql`(SELECT ROUND(page_count * page_size / (1024.0 * 1024 * 1024), 4) FROM pragma_page_count(), pragma_page_size())`.as("db_size_gb"),
289
+ totalTables: sql`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table')`.as("total_tables"),
290
+ totalIndexes: sql`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index')`.as("total_indexes"),
291
+ totalRows: sql`(SELECT SUM(tbl_rows) FROM (SELECT name, (SELECT COUNT(*) FROM sqlite_master sm WHERE sm.type='table' AND sm.name=t.name) AS tbl_rows FROM sqlite_master t WHERE type='table'))`.as("total_rows"),
292
+ pageSize: sql`(SELECT page_size FROM pragma_page_size())`.as("page_size"),
293
+ totalPages: sql`(SELECT page_count FROM pragma_page_count())`.as("total_pages")
294
+ }).from(sql`(SELECT 1) AS dummy`);
295
+ });
296
+ var rssdTableStatistic = sqliteView("rssd_table_statistic", {}).as(sql`
297
+ SELECT
298
+ m.name AS table_name,
299
+ (SELECT COUNT(*) FROM pragma_table_info(m.name)) AS total_columns,
300
+ (SELECT COUNT(*) FROM pragma_index_list(m.name)) AS total_indexes,
301
+ (SELECT COUNT(*) FROM pragma_foreign_key_list(m.name)) AS foreign_keys,
302
+ (SELECT COUNT(*) FROM pragma_table_info(m.name) WHERE pk != 0) AS primary_keys,
303
+ (SELECT table_size_mb FROM surveilr_table_size WHERE table_name = m.name) AS table_size_mb
304
+ FROM sqlite_master m
305
+ WHERE m.type = 'table'
306
+ `);
307
+ var uniformResourceFile = sqliteView("uniform_resource_file").as((qb) => {
308
+ return qb.select({
309
+ uniformResourceId: uniformResource.uniformResourceId,
310
+ nature: uniformResource.nature,
311
+ sourcePath: urIngestSessionFsPath.rootPath,
312
+ filePathRel: urIngestSessionFsPathEntry.filePathRel,
313
+ sizeBytes: uniformResource.sizeBytes
314
+ }).from(uniformResource).leftJoin(
315
+ uniformResourceEdge,
316
+ and(
317
+ eq(uniformResource.uniformResourceId, uniformResourceEdge.uniformResourceId),
318
+ eq(uniformResourceEdge.nature, "ingest_fs_path")
319
+ )
320
+ ).leftJoin(urIngestSessionFsPath, eq(uniformResourceEdge.nodeId, urIngestSessionFsPath.urIngestSessionFsPathId)).leftJoin(urIngestSessionFsPathEntry, eq(uniformResource.uniformResourceId, urIngestSessionFsPathEntry.uniformResourceId));
321
+ });
322
+ var uniformResourceImap = sqliteView("uniform_resource_imap").as((qb) => {
323
+ return qb.select({
324
+ uniformResourceId: uniformResource.uniformResourceId,
325
+ graphName: uniformResourceGraph.name,
326
+ urIngestSessionImapAccountId: urIngestSessionImapAccount.urIngestSessionImapAccountId,
327
+ email: urIngestSessionImapAccount.email,
328
+ host: urIngestSessionImapAccount.host,
329
+ subject: urIngestSessionImapAcctFolderMessage.subject,
330
+ from: urIngestSessionImapAcctFolderMessage.from,
331
+ message: urIngestSessionImapAcctFolderMessage.message,
332
+ date: urIngestSessionImapAcctFolderMessage.date,
333
+ urIngestSessionImapAcctFolderId: urIngestSessionImapAcctFolder.urIngestSessionImapAcctFolderId,
334
+ ingestAccountId: urIngestSessionImapAcctFolder.ingestAccountId,
335
+ folderName: urIngestSessionImapAcctFolder.folderName,
336
+ sizeBytes: uniformResource.sizeBytes,
337
+ nature: uniformResource.nature,
338
+ content: uniformResource.content
339
+ }).from(uniformResource).innerJoin(uniformResourceEdge, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(uniformResourceGraph, eq(uniformResourceGraph.name, uniformResourceEdge.graphName)).innerJoin(urIngestSessionImapAcctFolderMessage, eq(urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId, uniformResourceEdge.nodeId)).innerJoin(urIngestSessionImapAcctFolder, eq(urIngestSessionImapAcctFolderMessage.ingestImapAcctFolderId, urIngestSessionImapAcctFolder.urIngestSessionImapAcctFolderId)).leftJoin(urIngestSessionImapAccount, eq(urIngestSessionImapAccount.urIngestSessionImapAccountId, urIngestSessionImapAcctFolder.ingestAccountId)).where(and(
340
+ eq(uniformResource.nature, "text"),
341
+ eq(uniformResourceGraph.name, "imap"),
342
+ sql`${uniformResource.ingestSessionImapAcctFolderMessage} IS NOT NULL`
343
+ ));
344
+ });
345
+ var uniformResourceImapContent = sqliteView("uniform_resource_imap_content").as((qb) => {
346
+ return qb.select({
347
+ uniformResourceId: uniformResourceImap.uniformResourceId,
348
+ baseId: sql`base_ur.uniform_resource_id`.as("base_id"),
349
+ extId: sql`ext_ur.uniform_resource_id`.as("ext_id"),
350
+ baseUri: sql`base_ur.uri`.as("base_uri"),
351
+ extUri: sql`ext_ur.uri`.as("ext_uri"),
352
+ baseNature: sql`base_ur.nature`.as("base_nature"),
353
+ extNature: sql`ext_ur.nature`.as("ext_nature"),
354
+ htmlContent: sql`json_extract(part.value, '$.body.Html')`.as("html_content")
355
+ }).from(uniformResourceImap).innerJoin(sql`uniform_resource base_ur`, sql`base_ur.uniform_resource_id = ${uniformResourceImap.uniformResourceId}`).innerJoin(sql`uniform_resource ext_ur`, sql`ext_ur.uri = base_ur.uri || '/json' AND ext_ur.nature = 'json'`).crossJoin(sql`json_each(ext_ur.content, '$.parts') AS part`).where(and(
356
+ sql`ext_ur.nature = 'json'`,
357
+ sql`json_extract(part.value, '$.body.Html') IS NOT NULL`
358
+ ));
359
+ });
360
+ var urIngestSessionFilesStats = sqliteView("ur_ingest_session_files_stats").as((qb) => {
361
+ return qb.select({
362
+ deviceId: device.deviceId,
363
+ ingestSessionId: urIngestSession.urIngestSessionId,
364
+ ingestSessionStartedAt: urIngestSession.ingestStartedAt,
365
+ ingestSessionFinishedAt: urIngestSession.ingestFinishedAt,
366
+ fileExtension: sql`COALESCE(${urIngestSessionFsPathEntry.fileExtn}, '')`.as("file_extension"),
367
+ ingestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
368
+ ingestSessionRootFsPath: urIngestSessionFsPath.rootPath,
369
+ totalFileCount: count(urIngestSessionFsPathEntry.uniformResourceId).as("total_file_count"),
370
+ fileCountWithContent: sum(sql`CASE WHEN ${uniformResource.content} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_content"),
371
+ fileCountWithFrontmatter: sum(sql`CASE WHEN ${uniformResource.frontmatter} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_frontmatter"),
372
+ minFileSizeBytes: min(uniformResource.sizeBytes).as("min_file_size_bytes"),
373
+ averageFileSizeBytes: sql`CAST(ROUND(AVG(${uniformResource.sizeBytes})) AS INTEGER)`.as("average_file_size_bytes"),
374
+ maxFileSizeBytes: max(uniformResource.sizeBytes).as("max_file_size_bytes"),
375
+ oldestFileLastModifiedDatetime: min(uniformResource.lastModifiedAt).as("oldest_file_last_modified_datetime"),
376
+ youngestFileLastModifiedDatetime: max(uniformResource.lastModifiedAt).as("youngest_file_last_modified_datetime")
377
+ }).from(urIngestSession).innerJoin(device, eq(urIngestSession.deviceId, device.deviceId)).leftJoin(urIngestSessionFsPath, eq(urIngestSession.urIngestSessionId, urIngestSessionFsPath.ingestSessionId)).leftJoin(urIngestSessionFsPathEntry, eq(urIngestSessionFsPath.urIngestSessionFsPathId, urIngestSessionFsPathEntry.ingestFsPathId)).leftJoin(uniformResource, eq(urIngestSessionFsPathEntry.uniformResourceId, uniformResource.uniformResourceId)).groupBy(
378
+ device.deviceId,
379
+ urIngestSession.urIngestSessionId,
380
+ urIngestSession.ingestStartedAt,
381
+ urIngestSession.ingestFinishedAt,
382
+ urIngestSessionFsPathEntry.fileExtn,
383
+ urIngestSessionFsPath.rootPath
384
+ ).orderBy(device.deviceId, urIngestSession.ingestFinishedAt, urIngestSessionFsPathEntry.fileExtn);
385
+ });
386
+ var urIngestSessionFilesStatsLatest = sqliteView("ur_ingest_session_files_stats_latest").as((qb) => {
387
+ return qb.select({
388
+ deviceId: urIngestSessionFilesStats.deviceId,
389
+ ingestSessionId: urIngestSessionFilesStats.ingestSessionId,
390
+ ingestSessionStartedAt: urIngestSessionFilesStats.ingestSessionStartedAt,
391
+ ingestSessionFinishedAt: urIngestSessionFilesStats.ingestSessionFinishedAt,
392
+ fileExtension: urIngestSessionFilesStats.fileExtension,
393
+ ingestSessionFsPathId: urIngestSessionFilesStats.ingestSessionFsPathId,
394
+ ingestSessionRootFsPath: urIngestSessionFilesStats.ingestSessionRootFsPath,
395
+ totalFileCount: urIngestSessionFilesStats.totalFileCount,
396
+ fileCountWithContent: urIngestSessionFilesStats.fileCountWithContent,
397
+ fileCountWithFrontmatter: urIngestSessionFilesStats.fileCountWithFrontmatter,
398
+ minFileSizeBytes: urIngestSessionFilesStats.minFileSizeBytes,
399
+ averageFileSizeBytes: urIngestSessionFilesStats.averageFileSizeBytes,
400
+ maxFileSizeBytes: urIngestSessionFilesStats.maxFileSizeBytes,
401
+ oldestFileLastModifiedDatetime: urIngestSessionFilesStats.oldestFileLastModifiedDatetime,
402
+ youngestFileLastModifiedDatetime: urIngestSessionFilesStats.youngestFileLastModifiedDatetime
403
+ }).from(urIngestSessionFilesStats).where(sql`${urIngestSessionFilesStats.ingestSessionId} IN (
404
+ SELECT ${urIngestSession.urIngestSessionId}
405
+ FROM ${urIngestSession}
406
+ ORDER BY ${urIngestSession.ingestFinishedAt} DESC
407
+ LIMIT 1
408
+ )`);
409
+ });
410
+ var urIngestSessionTasksStats = sqliteView("ur_ingest_session_tasks_stats").as((qb) => {
411
+ return qb.select({
412
+ deviceId: device.deviceId,
413
+ ingestSessionId: urIngestSession.urIngestSessionId,
414
+ ingestSessionStartedAt: urIngestSession.ingestStartedAt,
415
+ ingestSessionFinishedAt: urIngestSession.ingestFinishedAt,
416
+ urStatus: sql`COALESCE(${urIngestSessionTask.urStatus}, 'Ok')`.as("ur_status"),
417
+ nature: sql`COALESCE(${uniformResource.nature}, 'UNKNOWN')`.as("nature"),
418
+ totalFileCount: count(urIngestSessionTask.uniformResourceId).as("total_file_count"),
419
+ fileCountWithContent: sum(sql`CASE WHEN ${uniformResource.content} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_content"),
420
+ fileCountWithFrontmatter: sum(sql`CASE WHEN ${uniformResource.frontmatter} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_frontmatter"),
421
+ minFileSizeBytes: min(uniformResource.sizeBytes).as("min_file_size_bytes"),
422
+ averageFileSizeBytes: sql`CAST(ROUND(AVG(${uniformResource.sizeBytes})) AS INTEGER)`.as("average_file_size_bytes"),
423
+ maxFileSizeBytes: max(uniformResource.sizeBytes).as("max_file_size_bytes"),
424
+ oldestFileLastModifiedDatetime: min(uniformResource.lastModifiedAt).as("oldest_file_last_modified_datetime"),
425
+ youngestFileLastModifiedDatetime: max(uniformResource.lastModifiedAt).as("youngest_file_last_modified_datetime")
426
+ }).from(urIngestSession).innerJoin(device, eq(urIngestSession.deviceId, device.deviceId)).leftJoin(urIngestSessionTask, eq(urIngestSession.urIngestSessionId, urIngestSessionTask.ingestSessionId)).leftJoin(uniformResource, eq(urIngestSessionTask.uniformResourceId, uniformResource.uniformResourceId)).groupBy(
427
+ device.deviceId,
428
+ urIngestSession.urIngestSessionId,
429
+ urIngestSession.ingestStartedAt,
430
+ urIngestSession.ingestFinishedAt,
431
+ urIngestSessionTask.capturedExecutable
432
+ ).orderBy(device.deviceId, urIngestSession.ingestFinishedAt, urIngestSessionTask.urStatus);
433
+ });
434
+ var urIngestSessionTasksStatsLatest = sqliteView("ur_ingest_session_tasks_stats_latest").as((qb) => {
435
+ return qb.select({
436
+ deviceId: urIngestSessionTasksStats.deviceId,
437
+ ingestSessionId: urIngestSessionTasksStats.ingestSessionId,
438
+ ingestSessionStartedAt: urIngestSessionTasksStats.ingestSessionStartedAt,
439
+ ingestSessionFinishedAt: urIngestSessionTasksStats.ingestSessionFinishedAt,
440
+ urStatus: urIngestSessionTasksStats.urStatus,
441
+ nature: urIngestSessionTasksStats.nature,
442
+ totalFileCount: urIngestSessionTasksStats.totalFileCount,
443
+ fileCountWithContent: urIngestSessionTasksStats.fileCountWithContent,
444
+ fileCountWithFrontmatter: urIngestSessionTasksStats.fileCountWithFrontmatter,
445
+ minFileSizeBytes: urIngestSessionTasksStats.minFileSizeBytes,
446
+ averageFileSizeBytes: urIngestSessionTasksStats.averageFileSizeBytes,
447
+ maxFileSizeBytes: urIngestSessionTasksStats.maxFileSizeBytes,
448
+ oldestFileLastModifiedDatetime: urIngestSessionTasksStats.oldestFileLastModifiedDatetime,
449
+ youngestFileLastModifiedDatetime: urIngestSessionTasksStats.youngestFileLastModifiedDatetime
450
+ }).from(urIngestSessionTasksStats).where(sql`${urIngestSessionTasksStats.ingestSessionId} IN (
451
+ SELECT ${urIngestSession.urIngestSessionId}
452
+ FROM ${urIngestSession}
453
+ ORDER BY ${urIngestSession.ingestFinishedAt} DESC
454
+ LIMIT 1
455
+ )`);
456
+ });
457
+ var urIngestSessionFileIssue = sqliteView("ur_ingest_session_file_issue").as((qb) => {
458
+ return qb.select({
459
+ deviceId: urIngestSession.deviceId,
460
+ urIngestSessionId: urIngestSession.urIngestSessionId,
461
+ urIngestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
462
+ rootPath: urIngestSessionFsPath.rootPath,
463
+ urIngestSessionFsPathEntryId: urIngestSessionFsPathEntry.urIngestSessionFsPathEntryId,
464
+ filePathAbs: urIngestSessionFsPathEntry.filePathAbs,
465
+ urStatus: urIngestSessionFsPathEntry.urStatus,
466
+ urDiagnostics: urIngestSessionFsPathEntry.urDiagnostics
467
+ }).from(urIngestSessionFsPathEntry).innerJoin(urIngestSessionFsPath, eq(urIngestSessionFsPathEntry.ingestFsPathId, urIngestSessionFsPath.urIngestSessionFsPathId)).innerJoin(urIngestSession, eq(urIngestSessionFsPath.ingestSessionId, urIngestSession.urIngestSessionId)).where(sql`${urIngestSessionFsPathEntry.urStatus} IS NOT NULL`).groupBy(
468
+ urIngestSession.deviceId,
469
+ urIngestSession.urIngestSessionId,
470
+ urIngestSessionFsPath.urIngestSessionFsPathId,
471
+ urIngestSessionFsPath.rootPath,
472
+ urIngestSessionFsPathEntry.urIngestSessionFsPathEntryId,
473
+ urIngestSessionFsPathEntry.filePathAbs,
474
+ urIngestSessionFsPathEntry.urStatus,
475
+ urIngestSessionFsPathEntry.urDiagnostics
476
+ );
477
+ });
478
+ var orchestrationSessionByDevice = sqliteView("orchestration_session_by_device").as((qb) => {
479
+ return qb.select({
480
+ deviceId: device.deviceId,
481
+ deviceName: device.name,
482
+ sessionCount: count(orchestrationSession.orchestrationSessionId).as("session_count")
483
+ }).from(orchestrationSession).innerJoin(device, eq(orchestrationSession.deviceId, device.deviceId)).groupBy(device.deviceId, device.name);
484
+ });
485
+ var orchestrationSessionDuration = sqliteView("orchestration_session_duration").as((qb) => {
486
+ return qb.select({
487
+ orchestrationSessionId: orchestrationSession.orchestrationSessionId,
488
+ orchestrationNature: orchestrationNature.nature,
489
+ orchStartedAt: orchestrationSession.orchStartedAt,
490
+ orchFinishedAt: orchestrationSession.orchFinishedAt,
491
+ durationSeconds: sql`(JULIANDAY(${orchestrationSession.orchFinishedAt}) - JULIANDAY(${orchestrationSession.orchStartedAt})) * 24 * 60 * 60`.as("duration_seconds")
492
+ }).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).where(sql`${orchestrationSession.orchFinishedAt} IS NOT NULL`);
493
+ });
494
+ var orchestrationSuccessRate = sqliteView("orchestration_success_rate").as((qb) => {
495
+ return qb.select({
496
+ orchestrationNature: orchestrationNature.nature,
497
+ totalSessions: count().as("total_sessions"),
498
+ successfulSessions: sum(sql`CASE WHEN ${orchestrationSessionState.toState} = 'surveilr_orch_completed' THEN 1 ELSE 0 END`).as("successful_sessions"),
499
+ successRate: sql`(CAST(SUM(CASE WHEN ${orchestrationSessionState.toState} = 'surveilr_orch_completed' THEN 1 ELSE 0 END) AS FLOAT) / COUNT(*)) * 100`.as("success_rate")
500
+ }).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionState, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionState.sessionId)).where(sql`${orchestrationSessionState.toState} IN ('surveilr_orch_completed', 'surveilr_orch_failed')`).groupBy(orchestrationNature.nature);
501
+ });
502
+ var orchestrationSessionScript = sqliteView("orchestration_session_script").as((qb) => {
503
+ return qb.select({
504
+ orchestrationSessionId: orchestrationSession.orchestrationSessionId,
505
+ orchestrationNature: orchestrationNature.nature,
506
+ scriptCount: count(orchestrationSessionEntry.orchestrationSessionEntryId).as("script_count")
507
+ }).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionEntry, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionEntry.sessionId)).groupBy(orchestrationSession.orchestrationSessionId, orchestrationNature.nature);
508
+ });
509
+ var orchestrationExecutionsByType = sqliteView("orchestration_executions_by_type").as((qb) => {
510
+ return qb.select({
511
+ execNature: orchestrationSessionExec.execNature,
512
+ executionCount: count(orchestrationSessionExec.orchestrationSessionExecId).as("execution_count")
513
+ }).from(orchestrationSessionExec).groupBy(orchestrationSessionExec.execNature);
514
+ });
515
+ var orchestrationExecutionSuccessRateByType = sqliteView("orchestration_execution_success_rate_by_type").as((qb) => {
516
+ return qb.select({
517
+ execNature: orchestrationSessionExec.execNature,
518
+ totalExecutions: count().as("total_executions"),
519
+ successfulExecutions: sum(sql`CASE WHEN ${orchestrationSessionExec.execStatus} = 0 THEN 1 ELSE 0 END`).as("successful_executions"),
520
+ successRate: sql`(CAST(SUM(CASE WHEN ${orchestrationSessionExec.execStatus} = 0 THEN 1 ELSE 0 END) AS FLOAT) / COUNT(*)) * 100`.as("success_rate")
521
+ }).from(orchestrationSessionExec).groupBy(orchestrationSessionExec.execNature);
522
+ });
523
+ var orchestrationSessionSummary = sqliteView("orchestration_session_summary").as((qb) => {
524
+ return qb.select({
525
+ issueType: orchestrationSessionIssue.issueType,
526
+ issueCount: count(orchestrationSessionIssue.orchestrationSessionIssueId).as("issue_count")
527
+ }).from(orchestrationSessionIssue).groupBy(orchestrationSessionIssue.issueType);
528
+ });
529
+ var orchestrationIssueRemediation = sqliteView("orchestration_issue_remediation").as((qb) => {
530
+ return qb.select({
531
+ orchestrationSessionIssueId: orchestrationSessionIssue.orchestrationSessionIssueId,
532
+ issueType: orchestrationSessionIssue.issueType,
533
+ issueMessage: orchestrationSessionIssue.issueMessage,
534
+ remediation: orchestrationSessionIssue.remediation
535
+ }).from(orchestrationSessionIssue).where(sql`${orchestrationSessionIssue.remediation} IS NOT NULL`);
536
+ });
537
+ var orchestrationLogsBySession = sqliteView("orchestration_logs_by_session").as((qb) => {
538
+ return qb.select({
539
+ orchestrationSessionId: orchestrationSession.orchestrationSessionId,
540
+ orchestrationNature: orchestrationNature.nature,
541
+ category: orchestrationSessionLog.category,
542
+ logCount: count(orchestrationSessionLog.orchestrationSessionLogId).as("log_count")
543
+ }).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionExec, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionExec.sessionId)).innerJoin(orchestrationSessionLog, eq(orchestrationSessionExec.orchestrationSessionExecId, orchestrationSessionLog.parentExecId)).groupBy(orchestrationSession.orchestrationSessionId, orchestrationNature.nature, orchestrationSessionLog.category);
544
+ });
545
+ var emailMessagesWithTimezone = sqliteView("email_messages_with_timezone", {}).as(sql`
546
+ SELECT
547
+ m.*,
548
+ -- Smart timezone conversion based on original timezone
549
+ CASE
550
+ -- If already IST (+0530), just extract the time
551
+ WHEN date LIKE '%+0530%' THEN substr(date, 18, 8)
552
+ -- If UTC (+0000), convert to IST
553
+ WHEN date LIKE '%+0000%' THEN time(substr(date, 18, 8), '+5 hours', '+30 minutes')
554
+ -- If EST (-0500), convert to IST
555
+ WHEN date LIKE '%-0500%' THEN time(substr(date, 18, 8), '+10 hours', '+30 minutes')
556
+ -- If PST (-0800), convert to IST
557
+ WHEN date LIKE '%-0800%' THEN time(substr(date, 18, 8), '+13 hours', '+30 minutes')
558
+ -- Default: assume UTC and convert
559
+ ELSE time(substr(date, 18, 8), '+5 hours', '+30 minutes')
560
+ END as ist_time,
561
+
562
+ -- Always show UTC time (convert from any timezone to UTC first)
563
+ CASE
564
+ WHEN date LIKE '%+0530%' THEN time(substr(date, 18, 8), '-5 hours', '-30 minutes')
565
+ WHEN date LIKE '%+0000%' THEN substr(date, 18, 8)
566
+ WHEN date LIKE '%-0500%' THEN time(substr(date, 18, 8), '+5 hours')
567
+ WHEN date LIKE '%-0800%' THEN time(substr(date, 18, 8), '+8 hours')
568
+ ELSE substr(date, 18, 8)
569
+ END as utc_time,
570
+
571
+ -- Detect original timezone more accurately
572
+ CASE
573
+ WHEN date LIKE '%+0530%' THEN 'IST'
574
+ WHEN date LIKE '%+0000%' THEN 'UTC'
575
+ WHEN date LIKE '%-0500%' THEN 'EST'
576
+ WHEN date LIKE '%-0800%' THEN 'PST'
577
+ ELSE 'Unknown'
578
+ END as detected_timezone
579
+ FROM ur_ingest_session_imap_acct_folder_message m`);
580
+ var deviceGraphOverview = sqliteView("device_graph_overview", {}).as(sql`
581
+ SELECT
582
+ 'snmp_device' as device_type,
583
+ sd.device_key as device_id,
584
+ sd.snmp_host as primary_identifier,
585
+ sd.device_type as device_category,
586
+ sd.device_description as description,
587
+ sd.status,
588
+ sd.created_at,
589
+ sd.updated_at,
590
+ json_object(
591
+ 'snmp_port', sd.snmp_port,
592
+ 'snmp_community', sd.snmp_community,
593
+ 'snmp_version', sd.snmp_version,
594
+ 'snmp_timeout', sd.snmp_timeout,
595
+ 'snmp_retries', sd.snmp_retries
596
+ ) as device_metadata
597
+ FROM surveilr_snmp_device sd
598
+ WHERE sd.status = 'active'
599
+
600
+ UNION ALL
601
+
602
+ SELECT
603
+ 'osquery_node' as device_type,
604
+ osq.node_key as device_id,
605
+ osq.host_identifier as primary_identifier,
606
+ 'endpoint' as device_category,
607
+ osq.os_version as description,
608
+ osq.status,
609
+ osq.created_at,
610
+ osq.updated_at,
611
+ json_object(
612
+ 'last_seen', osq.last_seen,
613
+ 'platform', osq.platform,
614
+ 'osquery_version', osq.osquery_version,
615
+ 'device_id', osq.device_id
616
+ ) as device_metadata
617
+ FROM surveilr_osquery_ms_node osq
618
+ WHERE osq.status = 'active'`);
619
+ var deviceGraphCorrelations = sqliteView("device_graph_correlations", {}).as(sql`
620
+ SELECT DISTINCT
621
+ ure1.node_id as snmp_device_key,
622
+ ure1.uniform_resource_id as osquery_node_key,
623
+ CASE
624
+ WHEN ure1.nature LIKE '%ip_address_match%' THEN 'IP Address Match'
625
+ WHEN ure1.nature LIKE '%hostname_match%' THEN 'Hostname Match'
626
+ WHEN ure1.nature LIKE '%mac_address_match%' THEN 'MAC Address Match'
627
+ WHEN ure1.nature LIKE '%network_topology%' THEN 'Network Topology'
628
+ WHEN ure1.nature LIKE '%manual%' THEN 'Manual'
629
+ ELSE 'Unknown'
630
+ END as correlation_type,
631
+ CASE
632
+ WHEN ure1.nature LIKE '%ip_address_match%' THEN 0.95
633
+ WHEN ure1.nature LIKE '%hostname_match%' THEN 0.85
634
+ WHEN ure1.nature LIKE '%mac_address_match%' THEN 0.90
635
+ WHEN ure1.nature LIKE '%network_topology%' THEN 0.70
636
+ WHEN ure1.nature LIKE '%manual%' THEN 1.0
637
+ ELSE 0.5
638
+ END as confidence_score,
639
+ json_extract(ure1.elaboration, '$.snmp_host') as snmp_host,
640
+ json_extract(ure1.elaboration, '$.correlation_method') as correlation_method,
641
+ ure1.elaboration as evidence
642
+ FROM uniform_resource_edge ure1
643
+ WHERE ure1.graph_name = 'device_graph'
644
+ AND ure1.nature LIKE 'snmp_to_osquery_%'
645
+ ORDER BY ure1.node_id, ure1.uniform_resource_id`);
646
+ var snmpDeviceInventory = sqliteView("snmp_device_inventory", {}).as(sql`
647
+ SELECT
648
+ sd.device_key,
649
+ sd.snmp_host,
650
+ sd.snmp_port,
651
+ sd.device_type,
652
+ sd.device_description,
653
+ sd.status,
654
+ sd.created_at,
655
+ COUNT(sc.surveilr_snmp_collection_id) as total_collections,
656
+ MAX(sc.collected_at) as last_collection_time,
657
+ COUNT(DISTINCT sc.oid) as unique_oids_collected,
658
+ AVG(
659
+ CASE
660
+ WHEN sc.collected_at > datetime('now', '-1 hour') THEN 1
661
+ ELSE 0
662
+ END
663
+ ) as recent_activity_score
664
+ FROM surveilr_snmp_device sd
665
+ LEFT JOIN surveilr_snmp_collection sc ON sd.device_key = sc.device_key
666
+ GROUP BY sd.device_key, sd.snmp_host, sd.snmp_port, sd.device_type,
667
+ sd.device_description, sd.status, sd.created_at
668
+ ORDER BY sd.status, sd.snmp_host`);
669
+ var snmpCollectionSummary = sqliteView("snmp_collection_summary", {}).as(sql`
670
+ SELECT
671
+ sc.device_key,
672
+ sd.snmp_host,
673
+ sc.oid_type,
674
+ COUNT(*) as collection_count,
675
+ COUNT(DISTINCT sc.oid) as unique_oids,
676
+ MIN(sc.collected_at) as first_collection,
677
+ MAX(sc.collected_at) as latest_collection,
678
+ COUNT(DISTINCT DATE(sc.collected_at)) as collection_days
679
+ FROM surveilr_snmp_collection sc
680
+ JOIN surveilr_snmp_device sd ON sc.device_key = sd.device_key
681
+ GROUP BY sc.device_key, sd.snmp_host, sc.oid_type
682
+ ORDER BY sd.snmp_host, sc.oid_type`);
683
+ var networkDeviceTopology = sqliteView("network_device_topology", {}).as(sql`
684
+ SELECT
685
+ sd.device_key,
686
+ sd.snmp_host,
687
+ sd.device_type,
688
+ -- Extract network information from SNMP collections
689
+ COALESCE(
690
+ (SELECT sc.oid_value
691
+ FROM surveilr_snmp_collection sc
692
+ WHERE sc.device_key = sd.device_key
693
+ AND sc.oid LIKE '%.1.3.6.1.2.1.1.1.%'
694
+ LIMIT 1),
695
+ 'Unknown'
696
+ ) as system_description,
697
+ COALESCE(
698
+ (SELECT sc.oid_value
699
+ FROM surveilr_snmp_collection sc
700
+ WHERE sc.device_key = sd.device_key
701
+ AND sc.oid LIKE '%.1.3.6.1.2.1.1.5.%'
702
+ LIMIT 1),
703
+ 'Unknown'
704
+ ) as system_name,
705
+ -- Count correlated devices
706
+ (SELECT COUNT(*)
707
+ FROM device_graph_correlations dgc
708
+ WHERE dgc.snmp_device_key = sd.device_key
709
+ ) as correlated_devices_count,
710
+ sd.status,
711
+ sd.created_at
712
+ FROM surveilr_snmp_device sd
713
+ WHERE sd.status = 'active'
714
+ ORDER BY sd.snmp_host`);
715
+ var deviceGraphAnalytics = sqliteView("device_graph_analytics", {}).as(sql`
716
+ SELECT
717
+ 'summary' as metric_type,
718
+ 'total_snmp_devices' as metric_name,
719
+ COUNT(*) as metric_value,
720
+ NULL as additional_info
721
+ FROM surveilr_snmp_device
722
+ WHERE status = 'active'
723
+
724
+ UNION ALL
725
+
726
+ SELECT
727
+ 'summary' as metric_type,
728
+ 'total_osquery_nodes' as metric_name,
729
+ COUNT(*) as metric_value,
730
+ NULL as additional_info
731
+ FROM surveilr_osquery_ms_node
732
+ WHERE status = 'active'
733
+
734
+ UNION ALL
735
+
736
+ SELECT
737
+ 'summary' as metric_type,
738
+ 'total_correlations' as metric_name,
739
+ COUNT(*) as metric_value,
740
+ NULL as additional_info
741
+ FROM device_graph_correlations
742
+
743
+ UNION ALL
744
+
745
+ SELECT
746
+ 'summary' as metric_type,
747
+ 'correlation_types' as metric_name,
748
+ COUNT(DISTINCT correlation_type) as metric_value,
749
+ GROUP_CONCAT(DISTINCT correlation_type) as additional_info
750
+ FROM device_graph_correlations
751
+
752
+ UNION ALL
753
+
754
+ SELECT
755
+ 'correlation_by_type' as metric_type,
756
+ correlation_type as metric_name,
757
+ COUNT(*) as metric_value,
758
+ printf('avg_confidence: %.2f', AVG(confidence_score)) as additional_info
759
+ FROM device_graph_correlations
760
+ GROUP BY correlation_type
761
+
762
+ UNION ALL
763
+
764
+ SELECT
765
+ 'collection_stats' as metric_type,
766
+ 'total_snmp_collections' as metric_name,
767
+ COUNT(*) as metric_value,
768
+ printf('unique_devices: %d', COUNT(DISTINCT device_key)) as additional_info
769
+ FROM surveilr_snmp_collection
770
+
771
+ UNION ALL
772
+
773
+ SELECT
774
+ 'collection_stats' as metric_type,
775
+ 'collection_activity_last_24h' as metric_name,
776
+ COUNT(*) as metric_value,
777
+ printf('devices_active: %d', COUNT(DISTINCT device_key)) as additional_info
778
+ FROM surveilr_snmp_collection
779
+ WHERE collected_at > datetime('now', '-24 hours')`);
780
+ var snmpOidPerformance = sqliteView("snmp_oid_performance", {}).as(sql`
781
+ SELECT
782
+ sc.oid,
783
+ sc.oid_type,
784
+ COUNT(*) as total_collections,
785
+ COUNT(DISTINCT sc.device_key) as devices_collecting,
786
+ COUNT(DISTINCT DATE(sc.collected_at)) as collection_days,
787
+ MIN(sc.collected_at) as first_seen,
788
+ MAX(sc.collected_at) as last_seen,
789
+ -- Calculate collection frequency (avoid nested aggregates)
790
+ CAST(
791
+ COUNT(*) * 1.0 /
792
+ NULLIF(MAX(julianday(sc.collected_at)) - MIN(julianday(sc.collected_at)), 0)
793
+ AS REAL) as avg_collections_per_day,
794
+ -- Sample values for diversity analysis
795
+ COUNT(DISTINCT sc.oid_value) as unique_values_count,
796
+ GROUP_CONCAT(DISTINCT sc.oid_value) as sample_values
797
+ FROM surveilr_snmp_collection sc
798
+ GROUP BY sc.oid, sc.oid_type
799
+ HAVING COUNT(*) > 0
800
+ ORDER BY total_collections DESC, devices_collecting DESC`);
801
+
802
+ export {
803
+ codeNotebookCellVersions,
804
+ codeNotebookCellLatest,
805
+ codeNotebookSqlCellMigratableVersion,
806
+ codeNotebookSqlCellMigratable,
807
+ codeNotebookSqlCellMigratableState,
808
+ codeNotebookSqlCellMigratableNotExecuted,
809
+ codeNotebookMigrationSql,
810
+ imapGraph,
811
+ filesystemGraph,
812
+ consoleInformationSchemaTable,
813
+ consoleInformationSchemaView,
814
+ consoleContentTabular,
815
+ consoleInformationSchemaTableColFkey,
816
+ consoleInformationSchemaTableColIndex,
817
+ rssdStatisticsOverview,
818
+ rssdTableStatistic,
819
+ uniformResourceFile,
820
+ uniformResourceImap,
821
+ uniformResourceImapContent,
822
+ urIngestSessionFilesStats,
823
+ urIngestSessionFilesStatsLatest,
824
+ urIngestSessionTasksStats,
825
+ urIngestSessionTasksStatsLatest,
826
+ urIngestSessionFileIssue,
827
+ orchestrationSessionByDevice,
828
+ orchestrationSessionDuration,
829
+ orchestrationSuccessRate,
830
+ orchestrationSessionScript,
831
+ orchestrationExecutionsByType,
832
+ orchestrationExecutionSuccessRateByType,
833
+ orchestrationSessionSummary,
834
+ orchestrationIssueRemediation,
835
+ orchestrationLogsBySession,
836
+ emailMessagesWithTimezone,
837
+ deviceGraphOverview,
838
+ deviceGraphCorrelations,
839
+ snmpDeviceInventory,
840
+ snmpCollectionSummary,
841
+ networkDeviceTopology,
842
+ deviceGraphAnalytics,
843
+ snmpOidPerformance
844
+ };