@surveilr/bootstrap-sql 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/chunk-HZLAQ5XF.js +1315 -0
- package/dist/chunk-QTY32BJY.js +844 -0
- package/dist/index.cjs +2896 -0
- package/dist/index.d.cts +278 -0
- package/dist/index.d.ts +278 -0
- package/dist/index.js +808 -0
- package/dist/models.cjs +1394 -0
- package/dist/models.d.cts +16708 -0
- package/dist/models.d.ts +16708 -0
- package/dist/models.js +140 -0
- package/dist/views.cjs +2120 -0
- package/dist/views.d.cts +2462 -0
- package/dist/views.d.ts +2462 -0
- package/dist/views.js +87 -0
- package/package.json +51 -0
|
@@ -0,0 +1,844 @@
|
|
|
1
|
+
import {
|
|
2
|
+
codeNotebookCell,
|
|
3
|
+
codeNotebookState,
|
|
4
|
+
device,
|
|
5
|
+
orchestrationNature,
|
|
6
|
+
orchestrationSession,
|
|
7
|
+
orchestrationSessionEntry,
|
|
8
|
+
orchestrationSessionExec,
|
|
9
|
+
orchestrationSessionIssue,
|
|
10
|
+
orchestrationSessionLog,
|
|
11
|
+
orchestrationSessionState,
|
|
12
|
+
uniformResource,
|
|
13
|
+
uniformResourceEdge,
|
|
14
|
+
uniformResourceGraph,
|
|
15
|
+
urIngestSession,
|
|
16
|
+
urIngestSessionFsPath,
|
|
17
|
+
urIngestSessionFsPathEntry,
|
|
18
|
+
urIngestSessionImapAccount,
|
|
19
|
+
urIngestSessionImapAcctFolder,
|
|
20
|
+
urIngestSessionImapAcctFolderMessage,
|
|
21
|
+
urIngestSessionTask
|
|
22
|
+
} from "./chunk-HZLAQ5XF.js";
|
|
23
|
+
|
|
24
|
+
// views.ts
|
|
25
|
+
import { sqliteView } from "drizzle-orm/sqlite-core";
|
|
26
|
+
import { sql, count, min, max, sum, eq, and } from "drizzle-orm";
|
|
27
|
+
var codeNotebookCellVersions = sqliteView("code_notebook_cell_versions").as((qb) => {
|
|
28
|
+
return qb.select({
|
|
29
|
+
notebookName: codeNotebookCell.notebookName,
|
|
30
|
+
notebookKernelId: codeNotebookCell.notebookKernelId,
|
|
31
|
+
cellName: codeNotebookCell.cellName,
|
|
32
|
+
versions: sql`COUNT(*) OVER(PARTITION BY ${codeNotebookCell.notebookName}, ${codeNotebookCell.cellName})`.as("versions"),
|
|
33
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId
|
|
34
|
+
}).from(codeNotebookCell).orderBy(codeNotebookCell.notebookName, codeNotebookCell.cellName);
|
|
35
|
+
});
|
|
36
|
+
var codeNotebookCellLatest = sqliteView("code_notebook_cell_latest").as((qb) => {
|
|
37
|
+
return qb.select({
|
|
38
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
|
|
39
|
+
notebookKernelId: codeNotebookCell.notebookKernelId,
|
|
40
|
+
notebookName: codeNotebookCell.notebookName,
|
|
41
|
+
cellName: codeNotebookCell.cellName,
|
|
42
|
+
interpretableCode: codeNotebookCell.interpretableCode,
|
|
43
|
+
interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
|
|
44
|
+
description: codeNotebookCell.description,
|
|
45
|
+
cellGovernance: codeNotebookCell.cellGovernance,
|
|
46
|
+
arguments: codeNotebookCell.arguments,
|
|
47
|
+
activityLog: codeNotebookCell.activityLog,
|
|
48
|
+
versionTimestamp: sql`COALESCE(${codeNotebookCell.updatedAt}, ${codeNotebookCell.createdAt})`.as("version_timestamp")
|
|
49
|
+
}).from(codeNotebookCell).where(sql`${codeNotebookCell.codeNotebookCellId} IN (
|
|
50
|
+
SELECT code_notebook_cell_id
|
|
51
|
+
FROM (
|
|
52
|
+
SELECT code_notebook_cell_id,
|
|
53
|
+
ROW_NUMBER() OVER (
|
|
54
|
+
PARTITION BY code_notebook_cell_id
|
|
55
|
+
ORDER BY COALESCE(updated_at, created_at) DESC
|
|
56
|
+
) AS rn
|
|
57
|
+
FROM code_notebook_cell
|
|
58
|
+
) ranked WHERE rn = 1
|
|
59
|
+
)`);
|
|
60
|
+
});
|
|
61
|
+
var codeNotebookSqlCellMigratableVersion = sqliteView("code_notebook_sql_cell_migratable_version").as((qb) => {
|
|
62
|
+
return qb.select({
|
|
63
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
|
|
64
|
+
notebookName: codeNotebookCell.notebookName,
|
|
65
|
+
cellName: codeNotebookCell.cellName,
|
|
66
|
+
interpretableCode: codeNotebookCell.interpretableCode,
|
|
67
|
+
interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
|
|
68
|
+
isIdempotent: sql`CASE WHEN ${codeNotebookCell.cellName} LIKE '%_once_%' THEN FALSE ELSE TRUE END`.as("is_idempotent"),
|
|
69
|
+
versionTimestamp: sql`COALESCE(${codeNotebookCell.updatedAt}, ${codeNotebookCell.createdAt})`.as("version_timestamp")
|
|
70
|
+
}).from(codeNotebookCell).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
|
|
71
|
+
});
|
|
72
|
+
var codeNotebookSqlCellMigratable = sqliteView("code_notebook_sql_cell_migratable").as((qb) => {
|
|
73
|
+
return qb.select({
|
|
74
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
|
|
75
|
+
notebookKernelId: codeNotebookCell.notebookKernelId,
|
|
76
|
+
notebookName: codeNotebookCell.notebookName,
|
|
77
|
+
cellName: codeNotebookCell.cellName,
|
|
78
|
+
interpretableCode: codeNotebookCell.interpretableCode,
|
|
79
|
+
interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
|
|
80
|
+
description: codeNotebookCell.description,
|
|
81
|
+
cellGovernance: codeNotebookCell.cellGovernance,
|
|
82
|
+
arguments: codeNotebookCell.arguments,
|
|
83
|
+
activityLog: codeNotebookCell.activityLog,
|
|
84
|
+
isIdempotent: sql`CASE WHEN ${codeNotebookCell.cellName} LIKE '%_once_%' THEN FALSE ELSE TRUE END`.as("is_idempotent")
|
|
85
|
+
}).from(codeNotebookCell).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
|
|
86
|
+
});
|
|
87
|
+
var codeNotebookSqlCellMigratableState = sqliteView("code_notebook_sql_cell_migratable_state").as((qb) => {
|
|
88
|
+
return qb.select({
|
|
89
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
|
|
90
|
+
notebookKernelId: codeNotebookCell.notebookKernelId,
|
|
91
|
+
notebookName: codeNotebookCell.notebookName,
|
|
92
|
+
cellName: codeNotebookCell.cellName,
|
|
93
|
+
interpretableCode: codeNotebookCell.interpretableCode,
|
|
94
|
+
interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
|
|
95
|
+
description: codeNotebookCell.description,
|
|
96
|
+
cellGovernance: codeNotebookCell.cellGovernance,
|
|
97
|
+
arguments: codeNotebookCell.arguments,
|
|
98
|
+
activityLog: codeNotebookCell.activityLog,
|
|
99
|
+
fromState: codeNotebookState.fromState,
|
|
100
|
+
toState: codeNotebookState.toState,
|
|
101
|
+
transitionReason: codeNotebookState.transitionReason,
|
|
102
|
+
transitionResult: codeNotebookState.transitionResult,
|
|
103
|
+
transitionedAt: codeNotebookState.transitionedAt
|
|
104
|
+
}).from(codeNotebookCell).innerJoin(codeNotebookState, eq(codeNotebookCell.codeNotebookCellId, codeNotebookState.codeNotebookCellId)).where(eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook")).orderBy(codeNotebookCell.cellName);
|
|
105
|
+
});
|
|
106
|
+
var codeNotebookSqlCellMigratableNotExecuted = sqliteView("code_notebook_sql_cell_migratable_not_executed").as((qb) => {
|
|
107
|
+
return qb.select({
|
|
108
|
+
codeNotebookCellId: codeNotebookCell.codeNotebookCellId,
|
|
109
|
+
notebookKernelId: codeNotebookCell.notebookKernelId,
|
|
110
|
+
notebookName: codeNotebookCell.notebookName,
|
|
111
|
+
cellName: codeNotebookCell.cellName,
|
|
112
|
+
interpretableCode: codeNotebookCell.interpretableCode,
|
|
113
|
+
interpretableCodeHash: codeNotebookCell.interpretableCodeHash,
|
|
114
|
+
description: codeNotebookCell.description,
|
|
115
|
+
cellGovernance: codeNotebookCell.cellGovernance,
|
|
116
|
+
arguments: codeNotebookCell.arguments,
|
|
117
|
+
activityLog: codeNotebookCell.activityLog
|
|
118
|
+
}).from(codeNotebookCell).leftJoin(codeNotebookState, and(
|
|
119
|
+
eq(codeNotebookCell.codeNotebookCellId, codeNotebookState.codeNotebookCellId),
|
|
120
|
+
eq(codeNotebookState.toState, "EXECUTED")
|
|
121
|
+
)).where(
|
|
122
|
+
and(
|
|
123
|
+
eq(codeNotebookCell.notebookName, "ConstructionSqlNotebook"),
|
|
124
|
+
sql`${codeNotebookState.codeNotebookCellId} IS NULL`
|
|
125
|
+
)
|
|
126
|
+
).orderBy(codeNotebookCell.cellName);
|
|
127
|
+
});
|
|
128
|
+
var codeNotebookMigrationSql = sqliteView("code_notebook_migration_sql", {}).as(sql`
|
|
129
|
+
SELECT 'BEGIN TRANSACTION;
|
|
130
|
+
|
|
131
|
+
' ||
|
|
132
|
+
'CREATE TEMP TABLE IF NOT EXISTS "session_state_ephemeral" (
|
|
133
|
+
"key" TEXT PRIMARY KEY NOT NULL,
|
|
134
|
+
"value" TEXT NOT NULL
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
' ||
|
|
138
|
+
GROUP_CONCAT(
|
|
139
|
+
CASE
|
|
140
|
+
-- Case 1: Non-idempotent and already executed
|
|
141
|
+
WHEN c.is_idempotent = FALSE AND s.code_notebook_cell_id IS NOT NULL THEN
|
|
142
|
+
'-- ' || c.notebook_name || '.' || c.cell_name || ' not included because it is non-idempotent and was already executed on ' || s.transitioned_at || '
|
|
143
|
+
|
|
144
|
+
'
|
|
145
|
+
-- Case 2: Idempotent and not yet executed, idempotent and being reapplied, or non-idempotent and being run for the first time
|
|
146
|
+
ELSE
|
|
147
|
+
'-- ' || c.notebook_name || '.' || c.cell_name || '
|
|
148
|
+
' ||
|
|
149
|
+
CASE
|
|
150
|
+
-- First execution (non-idempotent or idempotent)
|
|
151
|
+
WHEN s.code_notebook_cell_id IS NULL THEN
|
|
152
|
+
'-- Executing for the first time.
|
|
153
|
+
'
|
|
154
|
+
-- Reapplying execution (idempotent)
|
|
155
|
+
ELSE
|
|
156
|
+
'-- Reapplying execution. Last executed on ' || s.transitioned_at || '
|
|
157
|
+
'
|
|
158
|
+
END ||
|
|
159
|
+
c.interpretable_code || '
|
|
160
|
+
' ||
|
|
161
|
+
'INSERT INTO code_notebook_state (code_notebook_state_id, code_notebook_cell_id, from_state, to_state, transition_reason, created_at) ' ||
|
|
162
|
+
'VALUES (' ||
|
|
163
|
+
'''' || c.code_notebook_cell_id || '__' || strftime('%Y%m%d%H%M%S', 'now') || '''' || ', ' ||
|
|
164
|
+
'''' || c.code_notebook_cell_id || '''' || ', ' ||
|
|
165
|
+
'''MIGRATION_CANDIDATE''' || ', ' ||
|
|
166
|
+
'''EXECUTED''' || ', ' ||
|
|
167
|
+
CASE
|
|
168
|
+
WHEN s.code_notebook_cell_id IS NULL THEN '''Migration'''
|
|
169
|
+
ELSE '''Reapplication'''
|
|
170
|
+
END || ', ' ||
|
|
171
|
+
'CURRENT_TIMESTAMP' || ')' || '
|
|
172
|
+
' ||
|
|
173
|
+
'ON CONFLICT(code_notebook_cell_id, from_state, to_state) DO UPDATE SET updated_at = CURRENT_TIMESTAMP, ' ||
|
|
174
|
+
'transition_reason = ''Reapplied ' || datetime('now', 'localtime') || ''';' || '
|
|
175
|
+
'
|
|
176
|
+
END,
|
|
177
|
+
'
|
|
178
|
+
'
|
|
179
|
+
) || '
|
|
180
|
+
|
|
181
|
+
COMMIT;' as migration_sql
|
|
182
|
+
FROM code_notebook_sql_cell_migratable c
|
|
183
|
+
LEFT JOIN code_notebook_state s
|
|
184
|
+
ON c.code_notebook_cell_id = s.code_notebook_cell_id AND s.to_state = 'EXECUTED'
|
|
185
|
+
ORDER BY c.cell_name`);
|
|
186
|
+
var imapGraph = sqliteView("imap_graph").as((qb) => {
|
|
187
|
+
return qb.select({
|
|
188
|
+
graphName: uniformResourceEdge.graphName,
|
|
189
|
+
uniformResourceId: uniformResource.uniformResourceId,
|
|
190
|
+
nature: uniformResource.nature,
|
|
191
|
+
uri: uniformResource.uri,
|
|
192
|
+
content: uniformResource.content,
|
|
193
|
+
urIngestSessionImapAcctFolderMessageId: urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId,
|
|
194
|
+
ingestImapAcctFolderId: urIngestSessionImapAcctFolderMessage.ingestImapAcctFolderId,
|
|
195
|
+
messageId: urIngestSessionImapAcctFolderMessage.messageId
|
|
196
|
+
}).from(uniformResourceEdge).innerJoin(uniformResource, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(urIngestSessionImapAcctFolderMessage, eq(uniformResourceEdge.nodeId, urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId)).where(eq(uniformResourceEdge.graphName, "imap"));
|
|
197
|
+
});
|
|
198
|
+
var filesystemGraph = sqliteView("filesystem_graph").as((qb) => {
|
|
199
|
+
return qb.select({
|
|
200
|
+
graphName: uniformResourceEdge.graphName,
|
|
201
|
+
uniformResourceId: uniformResource.uniformResourceId,
|
|
202
|
+
nature: uniformResource.nature,
|
|
203
|
+
uri: uniformResource.uri,
|
|
204
|
+
urIngestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
|
|
205
|
+
rootPath: urIngestSessionFsPath.rootPath
|
|
206
|
+
}).from(uniformResourceEdge).innerJoin(uniformResource, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(urIngestSessionFsPath, eq(uniformResourceEdge.nodeId, urIngestSessionFsPath.urIngestSessionFsPathId)).where(eq(uniformResourceEdge.graphName, "filesystem"));
|
|
207
|
+
});
|
|
208
|
+
var consoleInformationSchemaTable = sqliteView("console_information_schema_table", {}).as(sql`
|
|
209
|
+
SELECT
|
|
210
|
+
tbl.name AS table_name,
|
|
211
|
+
col.name AS column_name,
|
|
212
|
+
col.type AS data_type,
|
|
213
|
+
CASE WHEN col.pk = 1 THEN 'Yes' ELSE 'No' END AS is_primary_key,
|
|
214
|
+
CASE WHEN col."notnull" = 1 THEN 'Yes' ELSE 'No' END AS is_not_null,
|
|
215
|
+
col.dflt_value AS default_value,
|
|
216
|
+
'console/info-schema/table.sql?name=' || tbl.name || '&stats=yes' as info_schema_web_ui_path,
|
|
217
|
+
'[Content](console/info-schema/table.sql?name=' || tbl.name || '&stats=yes)' as info_schema_link_abbrev_md,
|
|
218
|
+
'[' || tbl.name || ' (table) Schema](console/info-schema/table.sql?name=' || tbl.name || '&stats=yes)' as info_schema_link_full_md,
|
|
219
|
+
'console/content/table/' || tbl.name || '.sql?stats=yes' as content_web_ui_path,
|
|
220
|
+
'[Content]($SITE_PREFIX_URL/console/content/table/' || tbl.name || '.sql?stats=yes)' as content_web_ui_link_abbrev_md,
|
|
221
|
+
'[' || tbl.name || ' (table) Content](console/content/table/' || tbl.name || '.sql?stats=yes)' as content_web_ui_link_full_md,
|
|
222
|
+
tbl.sql as sql_ddl
|
|
223
|
+
FROM sqlite_master tbl
|
|
224
|
+
JOIN pragma_table_info(tbl.name) col
|
|
225
|
+
WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
|
|
226
|
+
`);
|
|
227
|
+
var consoleInformationSchemaView = sqliteView("console_information_schema_view", {}).as(sql`
|
|
228
|
+
SELECT
|
|
229
|
+
vw.name AS view_name,
|
|
230
|
+
col.name AS column_name,
|
|
231
|
+
col.type AS data_type,
|
|
232
|
+
'/console/info-schema/view.sql?name=' || vw.name || '&stats=yes' as info_schema_web_ui_path,
|
|
233
|
+
'[Content](console/info-schema/view.sql?name=' || vw.name || '&stats=yes)' as info_schema_link_abbrev_md,
|
|
234
|
+
'[' || vw.name || ' (view) Schema](console/info-schema/view.sql?name=' || vw.name || '&stats=yes)' as info_schema_link_full_md,
|
|
235
|
+
'/console/content/view/' || vw.name || '.sql?stats=yes' as content_web_ui_path,
|
|
236
|
+
'[Content]($SITE_PREFIX_URL/console/content/view/' || vw.name || '.sql?stats=yes)' as content_web_ui_link_abbrev_md,
|
|
237
|
+
'[' || vw.name || ' (view) Content](console/content/view/' || vw.name || '.sql?stats=yes)' as content_web_ui_link_full_md,
|
|
238
|
+
vw.sql as sql_ddl
|
|
239
|
+
FROM sqlite_master vw
|
|
240
|
+
JOIN pragma_table_info(vw.name) col
|
|
241
|
+
WHERE vw.type = 'view' AND vw.name NOT LIKE 'sqlite_%'
|
|
242
|
+
`);
|
|
243
|
+
var consoleContentTabular = sqliteView("console_content_tabular").as((qb) => {
|
|
244
|
+
const tableQuery = qb.select({
|
|
245
|
+
tabularNature: sql`'table'`.as("tabular_nature"),
|
|
246
|
+
tabularName: sql`table_name`.as("tabular_name"),
|
|
247
|
+
infoSchemaWebUiPath: sql`info_schema_web_ui_path`.as("info_schema_web_ui_path"),
|
|
248
|
+
infoSchemaLinkAbbrevMd: sql`info_schema_link_abbrev_md`.as("info_schema_link_abbrev_md"),
|
|
249
|
+
infoSchemaLinkFullMd: sql`info_schema_link_full_md`.as("info_schema_link_full_md"),
|
|
250
|
+
contentWebUiPath: sql`content_web_ui_path`.as("content_web_ui_path"),
|
|
251
|
+
contentWebUiLinkAbbrevMd: sql`content_web_ui_link_abbrev_md`.as("content_web_ui_link_abbrev_md"),
|
|
252
|
+
contentWebUiLinkFullMd: sql`content_web_ui_link_full_md`.as("content_web_ui_link_full_md")
|
|
253
|
+
}).from(consoleInformationSchemaTable);
|
|
254
|
+
const viewQuery = qb.select({
|
|
255
|
+
tabularNature: sql`'view'`.as("tabular_nature"),
|
|
256
|
+
tabularName: sql`view_name`.as("tabular_name"),
|
|
257
|
+
infoSchemaWebUiPath: sql`info_schema_web_ui_path`.as("info_schema_web_ui_path"),
|
|
258
|
+
infoSchemaLinkAbbrevMd: sql`info_schema_link_abbrev_md`.as("info_schema_link_abbrev_md"),
|
|
259
|
+
infoSchemaLinkFullMd: sql`info_schema_link_full_md`.as("info_schema_link_full_md"),
|
|
260
|
+
contentWebUiPath: sql`content_web_ui_path`.as("content_web_ui_path"),
|
|
261
|
+
contentWebUiLinkAbbrevMd: sql`content_web_ui_link_abbrev_md`.as("content_web_ui_link_abbrev_md"),
|
|
262
|
+
contentWebUiLinkFullMd: sql`content_web_ui_link_full_md`.as("content_web_ui_link_full_md")
|
|
263
|
+
}).from(consoleInformationSchemaView);
|
|
264
|
+
return tableQuery.unionAll(viewQuery);
|
|
265
|
+
});
|
|
266
|
+
var consoleInformationSchemaTableColFkey = sqliteView("console_information_schema_table_col_fkey", {}).as(sql`
|
|
267
|
+
SELECT
|
|
268
|
+
tbl.name AS table_name,
|
|
269
|
+
f."from" AS column_name,
|
|
270
|
+
f."from" || ' references ' || f."table" || '.' || f."to" AS foreign_key
|
|
271
|
+
FROM sqlite_master tbl
|
|
272
|
+
JOIN pragma_foreign_key_list(tbl.name) f
|
|
273
|
+
WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
|
|
274
|
+
`);
|
|
275
|
+
var consoleInformationSchemaTableColIndex = sqliteView("console_information_schema_table_col_index", {}).as(sql`
|
|
276
|
+
SELECT
|
|
277
|
+
tbl.name AS table_name,
|
|
278
|
+
pi.name AS column_name,
|
|
279
|
+
idx.name AS index_name
|
|
280
|
+
FROM sqlite_master tbl
|
|
281
|
+
JOIN pragma_index_list(tbl.name) idx
|
|
282
|
+
JOIN pragma_index_info(idx.name) pi
|
|
283
|
+
WHERE tbl.type = 'table' AND tbl.name NOT LIKE 'sqlite_%'
|
|
284
|
+
`);
|
|
285
|
+
var rssdStatisticsOverview = sqliteView("rssd_statistics_overview").as((qb) => {
|
|
286
|
+
return qb.select({
|
|
287
|
+
dbSizeMb: sql`(SELECT ROUND(page_count * page_size / (1024.0 * 1024), 2) FROM pragma_page_count(), pragma_page_size())`.as("db_size_mb"),
|
|
288
|
+
dbSizeGb: sql`(SELECT ROUND(page_count * page_size / (1024.0 * 1024 * 1024), 4) FROM pragma_page_count(), pragma_page_size())`.as("db_size_gb"),
|
|
289
|
+
totalTables: sql`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'table')`.as("total_tables"),
|
|
290
|
+
totalIndexes: sql`(SELECT COUNT(*) FROM sqlite_master WHERE type = 'index')`.as("total_indexes"),
|
|
291
|
+
totalRows: sql`(SELECT SUM(tbl_rows) FROM (SELECT name, (SELECT COUNT(*) FROM sqlite_master sm WHERE sm.type='table' AND sm.name=t.name) AS tbl_rows FROM sqlite_master t WHERE type='table'))`.as("total_rows"),
|
|
292
|
+
pageSize: sql`(SELECT page_size FROM pragma_page_size())`.as("page_size"),
|
|
293
|
+
totalPages: sql`(SELECT page_count FROM pragma_page_count())`.as("total_pages")
|
|
294
|
+
}).from(sql`(SELECT 1) AS dummy`);
|
|
295
|
+
});
|
|
296
|
+
var rssdTableStatistic = sqliteView("rssd_table_statistic", {}).as(sql`
|
|
297
|
+
SELECT
|
|
298
|
+
m.name AS table_name,
|
|
299
|
+
(SELECT COUNT(*) FROM pragma_table_info(m.name)) AS total_columns,
|
|
300
|
+
(SELECT COUNT(*) FROM pragma_index_list(m.name)) AS total_indexes,
|
|
301
|
+
(SELECT COUNT(*) FROM pragma_foreign_key_list(m.name)) AS foreign_keys,
|
|
302
|
+
(SELECT COUNT(*) FROM pragma_table_info(m.name) WHERE pk != 0) AS primary_keys,
|
|
303
|
+
(SELECT table_size_mb FROM surveilr_table_size WHERE table_name = m.name) AS table_size_mb
|
|
304
|
+
FROM sqlite_master m
|
|
305
|
+
WHERE m.type = 'table'
|
|
306
|
+
`);
|
|
307
|
+
var uniformResourceFile = sqliteView("uniform_resource_file").as((qb) => {
|
|
308
|
+
return qb.select({
|
|
309
|
+
uniformResourceId: uniformResource.uniformResourceId,
|
|
310
|
+
nature: uniformResource.nature,
|
|
311
|
+
sourcePath: urIngestSessionFsPath.rootPath,
|
|
312
|
+
filePathRel: urIngestSessionFsPathEntry.filePathRel,
|
|
313
|
+
sizeBytes: uniformResource.sizeBytes
|
|
314
|
+
}).from(uniformResource).leftJoin(
|
|
315
|
+
uniformResourceEdge,
|
|
316
|
+
and(
|
|
317
|
+
eq(uniformResource.uniformResourceId, uniformResourceEdge.uniformResourceId),
|
|
318
|
+
eq(uniformResourceEdge.nature, "ingest_fs_path")
|
|
319
|
+
)
|
|
320
|
+
).leftJoin(urIngestSessionFsPath, eq(uniformResourceEdge.nodeId, urIngestSessionFsPath.urIngestSessionFsPathId)).leftJoin(urIngestSessionFsPathEntry, eq(uniformResource.uniformResourceId, urIngestSessionFsPathEntry.uniformResourceId));
|
|
321
|
+
});
|
|
322
|
+
var uniformResourceImap = sqliteView("uniform_resource_imap").as((qb) => {
|
|
323
|
+
return qb.select({
|
|
324
|
+
uniformResourceId: uniformResource.uniformResourceId,
|
|
325
|
+
graphName: uniformResourceGraph.name,
|
|
326
|
+
urIngestSessionImapAccountId: urIngestSessionImapAccount.urIngestSessionImapAccountId,
|
|
327
|
+
email: urIngestSessionImapAccount.email,
|
|
328
|
+
host: urIngestSessionImapAccount.host,
|
|
329
|
+
subject: urIngestSessionImapAcctFolderMessage.subject,
|
|
330
|
+
from: urIngestSessionImapAcctFolderMessage.from,
|
|
331
|
+
message: urIngestSessionImapAcctFolderMessage.message,
|
|
332
|
+
date: urIngestSessionImapAcctFolderMessage.date,
|
|
333
|
+
urIngestSessionImapAcctFolderId: urIngestSessionImapAcctFolder.urIngestSessionImapAcctFolderId,
|
|
334
|
+
ingestAccountId: urIngestSessionImapAcctFolder.ingestAccountId,
|
|
335
|
+
folderName: urIngestSessionImapAcctFolder.folderName,
|
|
336
|
+
sizeBytes: uniformResource.sizeBytes,
|
|
337
|
+
nature: uniformResource.nature,
|
|
338
|
+
content: uniformResource.content
|
|
339
|
+
}).from(uniformResource).innerJoin(uniformResourceEdge, eq(uniformResourceEdge.uniformResourceId, uniformResource.uniformResourceId)).innerJoin(uniformResourceGraph, eq(uniformResourceGraph.name, uniformResourceEdge.graphName)).innerJoin(urIngestSessionImapAcctFolderMessage, eq(urIngestSessionImapAcctFolderMessage.urIngestSessionImapAcctFolderMessageId, uniformResourceEdge.nodeId)).innerJoin(urIngestSessionImapAcctFolder, eq(urIngestSessionImapAcctFolderMessage.ingestImapAcctFolderId, urIngestSessionImapAcctFolder.urIngestSessionImapAcctFolderId)).leftJoin(urIngestSessionImapAccount, eq(urIngestSessionImapAccount.urIngestSessionImapAccountId, urIngestSessionImapAcctFolder.ingestAccountId)).where(and(
|
|
340
|
+
eq(uniformResource.nature, "text"),
|
|
341
|
+
eq(uniformResourceGraph.name, "imap"),
|
|
342
|
+
sql`${uniformResource.ingestSessionImapAcctFolderMessage} IS NOT NULL`
|
|
343
|
+
));
|
|
344
|
+
});
|
|
345
|
+
var uniformResourceImapContent = sqliteView("uniform_resource_imap_content").as((qb) => {
|
|
346
|
+
return qb.select({
|
|
347
|
+
uniformResourceId: uniformResourceImap.uniformResourceId,
|
|
348
|
+
baseId: sql`base_ur.uniform_resource_id`.as("base_id"),
|
|
349
|
+
extId: sql`ext_ur.uniform_resource_id`.as("ext_id"),
|
|
350
|
+
baseUri: sql`base_ur.uri`.as("base_uri"),
|
|
351
|
+
extUri: sql`ext_ur.uri`.as("ext_uri"),
|
|
352
|
+
baseNature: sql`base_ur.nature`.as("base_nature"),
|
|
353
|
+
extNature: sql`ext_ur.nature`.as("ext_nature"),
|
|
354
|
+
htmlContent: sql`json_extract(part.value, '$.body.Html')`.as("html_content")
|
|
355
|
+
}).from(uniformResourceImap).innerJoin(sql`uniform_resource base_ur`, sql`base_ur.uniform_resource_id = ${uniformResourceImap.uniformResourceId}`).innerJoin(sql`uniform_resource ext_ur`, sql`ext_ur.uri = base_ur.uri || '/json' AND ext_ur.nature = 'json'`).crossJoin(sql`json_each(ext_ur.content, '$.parts') AS part`).where(and(
|
|
356
|
+
sql`ext_ur.nature = 'json'`,
|
|
357
|
+
sql`json_extract(part.value, '$.body.Html') IS NOT NULL`
|
|
358
|
+
));
|
|
359
|
+
});
|
|
360
|
+
var urIngestSessionFilesStats = sqliteView("ur_ingest_session_files_stats").as((qb) => {
|
|
361
|
+
return qb.select({
|
|
362
|
+
deviceId: device.deviceId,
|
|
363
|
+
ingestSessionId: urIngestSession.urIngestSessionId,
|
|
364
|
+
ingestSessionStartedAt: urIngestSession.ingestStartedAt,
|
|
365
|
+
ingestSessionFinishedAt: urIngestSession.ingestFinishedAt,
|
|
366
|
+
fileExtension: sql`COALESCE(${urIngestSessionFsPathEntry.fileExtn}, '')`.as("file_extension"),
|
|
367
|
+
ingestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
|
|
368
|
+
ingestSessionRootFsPath: urIngestSessionFsPath.rootPath,
|
|
369
|
+
totalFileCount: count(urIngestSessionFsPathEntry.uniformResourceId).as("total_file_count"),
|
|
370
|
+
fileCountWithContent: sum(sql`CASE WHEN ${uniformResource.content} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_content"),
|
|
371
|
+
fileCountWithFrontmatter: sum(sql`CASE WHEN ${uniformResource.frontmatter} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_frontmatter"),
|
|
372
|
+
minFileSizeBytes: min(uniformResource.sizeBytes).as("min_file_size_bytes"),
|
|
373
|
+
averageFileSizeBytes: sql`CAST(ROUND(AVG(${uniformResource.sizeBytes})) AS INTEGER)`.as("average_file_size_bytes"),
|
|
374
|
+
maxFileSizeBytes: max(uniformResource.sizeBytes).as("max_file_size_bytes"),
|
|
375
|
+
oldestFileLastModifiedDatetime: min(uniformResource.lastModifiedAt).as("oldest_file_last_modified_datetime"),
|
|
376
|
+
youngestFileLastModifiedDatetime: max(uniformResource.lastModifiedAt).as("youngest_file_last_modified_datetime")
|
|
377
|
+
}).from(urIngestSession).innerJoin(device, eq(urIngestSession.deviceId, device.deviceId)).leftJoin(urIngestSessionFsPath, eq(urIngestSession.urIngestSessionId, urIngestSessionFsPath.ingestSessionId)).leftJoin(urIngestSessionFsPathEntry, eq(urIngestSessionFsPath.urIngestSessionFsPathId, urIngestSessionFsPathEntry.ingestFsPathId)).leftJoin(uniformResource, eq(urIngestSessionFsPathEntry.uniformResourceId, uniformResource.uniformResourceId)).groupBy(
|
|
378
|
+
device.deviceId,
|
|
379
|
+
urIngestSession.urIngestSessionId,
|
|
380
|
+
urIngestSession.ingestStartedAt,
|
|
381
|
+
urIngestSession.ingestFinishedAt,
|
|
382
|
+
urIngestSessionFsPathEntry.fileExtn,
|
|
383
|
+
urIngestSessionFsPath.rootPath
|
|
384
|
+
).orderBy(device.deviceId, urIngestSession.ingestFinishedAt, urIngestSessionFsPathEntry.fileExtn);
|
|
385
|
+
});
|
|
386
|
+
var urIngestSessionFilesStatsLatest = sqliteView("ur_ingest_session_files_stats_latest").as((qb) => {
|
|
387
|
+
return qb.select({
|
|
388
|
+
deviceId: urIngestSessionFilesStats.deviceId,
|
|
389
|
+
ingestSessionId: urIngestSessionFilesStats.ingestSessionId,
|
|
390
|
+
ingestSessionStartedAt: urIngestSessionFilesStats.ingestSessionStartedAt,
|
|
391
|
+
ingestSessionFinishedAt: urIngestSessionFilesStats.ingestSessionFinishedAt,
|
|
392
|
+
fileExtension: urIngestSessionFilesStats.fileExtension,
|
|
393
|
+
ingestSessionFsPathId: urIngestSessionFilesStats.ingestSessionFsPathId,
|
|
394
|
+
ingestSessionRootFsPath: urIngestSessionFilesStats.ingestSessionRootFsPath,
|
|
395
|
+
totalFileCount: urIngestSessionFilesStats.totalFileCount,
|
|
396
|
+
fileCountWithContent: urIngestSessionFilesStats.fileCountWithContent,
|
|
397
|
+
fileCountWithFrontmatter: urIngestSessionFilesStats.fileCountWithFrontmatter,
|
|
398
|
+
minFileSizeBytes: urIngestSessionFilesStats.minFileSizeBytes,
|
|
399
|
+
averageFileSizeBytes: urIngestSessionFilesStats.averageFileSizeBytes,
|
|
400
|
+
maxFileSizeBytes: urIngestSessionFilesStats.maxFileSizeBytes,
|
|
401
|
+
oldestFileLastModifiedDatetime: urIngestSessionFilesStats.oldestFileLastModifiedDatetime,
|
|
402
|
+
youngestFileLastModifiedDatetime: urIngestSessionFilesStats.youngestFileLastModifiedDatetime
|
|
403
|
+
}).from(urIngestSessionFilesStats).where(sql`${urIngestSessionFilesStats.ingestSessionId} IN (
|
|
404
|
+
SELECT ${urIngestSession.urIngestSessionId}
|
|
405
|
+
FROM ${urIngestSession}
|
|
406
|
+
ORDER BY ${urIngestSession.ingestFinishedAt} DESC
|
|
407
|
+
LIMIT 1
|
|
408
|
+
)`);
|
|
409
|
+
});
|
|
410
|
+
var urIngestSessionTasksStats = sqliteView("ur_ingest_session_tasks_stats").as((qb) => {
|
|
411
|
+
return qb.select({
|
|
412
|
+
deviceId: device.deviceId,
|
|
413
|
+
ingestSessionId: urIngestSession.urIngestSessionId,
|
|
414
|
+
ingestSessionStartedAt: urIngestSession.ingestStartedAt,
|
|
415
|
+
ingestSessionFinishedAt: urIngestSession.ingestFinishedAt,
|
|
416
|
+
urStatus: sql`COALESCE(${urIngestSessionTask.urStatus}, 'Ok')`.as("ur_status"),
|
|
417
|
+
nature: sql`COALESCE(${uniformResource.nature}, 'UNKNOWN')`.as("nature"),
|
|
418
|
+
totalFileCount: count(urIngestSessionTask.uniformResourceId).as("total_file_count"),
|
|
419
|
+
fileCountWithContent: sum(sql`CASE WHEN ${uniformResource.content} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_content"),
|
|
420
|
+
fileCountWithFrontmatter: sum(sql`CASE WHEN ${uniformResource.frontmatter} IS NOT NULL THEN 1 ELSE 0 END`).as("file_count_with_frontmatter"),
|
|
421
|
+
minFileSizeBytes: min(uniformResource.sizeBytes).as("min_file_size_bytes"),
|
|
422
|
+
averageFileSizeBytes: sql`CAST(ROUND(AVG(${uniformResource.sizeBytes})) AS INTEGER)`.as("average_file_size_bytes"),
|
|
423
|
+
maxFileSizeBytes: max(uniformResource.sizeBytes).as("max_file_size_bytes"),
|
|
424
|
+
oldestFileLastModifiedDatetime: min(uniformResource.lastModifiedAt).as("oldest_file_last_modified_datetime"),
|
|
425
|
+
youngestFileLastModifiedDatetime: max(uniformResource.lastModifiedAt).as("youngest_file_last_modified_datetime")
|
|
426
|
+
}).from(urIngestSession).innerJoin(device, eq(urIngestSession.deviceId, device.deviceId)).leftJoin(urIngestSessionTask, eq(urIngestSession.urIngestSessionId, urIngestSessionTask.ingestSessionId)).leftJoin(uniformResource, eq(urIngestSessionTask.uniformResourceId, uniformResource.uniformResourceId)).groupBy(
|
|
427
|
+
device.deviceId,
|
|
428
|
+
urIngestSession.urIngestSessionId,
|
|
429
|
+
urIngestSession.ingestStartedAt,
|
|
430
|
+
urIngestSession.ingestFinishedAt,
|
|
431
|
+
urIngestSessionTask.capturedExecutable
|
|
432
|
+
).orderBy(device.deviceId, urIngestSession.ingestFinishedAt, urIngestSessionTask.urStatus);
|
|
433
|
+
});
|
|
434
|
+
var urIngestSessionTasksStatsLatest = sqliteView("ur_ingest_session_tasks_stats_latest").as((qb) => {
|
|
435
|
+
return qb.select({
|
|
436
|
+
deviceId: urIngestSessionTasksStats.deviceId,
|
|
437
|
+
ingestSessionId: urIngestSessionTasksStats.ingestSessionId,
|
|
438
|
+
ingestSessionStartedAt: urIngestSessionTasksStats.ingestSessionStartedAt,
|
|
439
|
+
ingestSessionFinishedAt: urIngestSessionTasksStats.ingestSessionFinishedAt,
|
|
440
|
+
urStatus: urIngestSessionTasksStats.urStatus,
|
|
441
|
+
nature: urIngestSessionTasksStats.nature,
|
|
442
|
+
totalFileCount: urIngestSessionTasksStats.totalFileCount,
|
|
443
|
+
fileCountWithContent: urIngestSessionTasksStats.fileCountWithContent,
|
|
444
|
+
fileCountWithFrontmatter: urIngestSessionTasksStats.fileCountWithFrontmatter,
|
|
445
|
+
minFileSizeBytes: urIngestSessionTasksStats.minFileSizeBytes,
|
|
446
|
+
averageFileSizeBytes: urIngestSessionTasksStats.averageFileSizeBytes,
|
|
447
|
+
maxFileSizeBytes: urIngestSessionTasksStats.maxFileSizeBytes,
|
|
448
|
+
oldestFileLastModifiedDatetime: urIngestSessionTasksStats.oldestFileLastModifiedDatetime,
|
|
449
|
+
youngestFileLastModifiedDatetime: urIngestSessionTasksStats.youngestFileLastModifiedDatetime
|
|
450
|
+
}).from(urIngestSessionTasksStats).where(sql`${urIngestSessionTasksStats.ingestSessionId} IN (
|
|
451
|
+
SELECT ${urIngestSession.urIngestSessionId}
|
|
452
|
+
FROM ${urIngestSession}
|
|
453
|
+
ORDER BY ${urIngestSession.ingestFinishedAt} DESC
|
|
454
|
+
LIMIT 1
|
|
455
|
+
)`);
|
|
456
|
+
});
|
|
457
|
+
var urIngestSessionFileIssue = sqliteView("ur_ingest_session_file_issue").as((qb) => {
|
|
458
|
+
return qb.select({
|
|
459
|
+
deviceId: urIngestSession.deviceId,
|
|
460
|
+
urIngestSessionId: urIngestSession.urIngestSessionId,
|
|
461
|
+
urIngestSessionFsPathId: urIngestSessionFsPath.urIngestSessionFsPathId,
|
|
462
|
+
rootPath: urIngestSessionFsPath.rootPath,
|
|
463
|
+
urIngestSessionFsPathEntryId: urIngestSessionFsPathEntry.urIngestSessionFsPathEntryId,
|
|
464
|
+
filePathAbs: urIngestSessionFsPathEntry.filePathAbs,
|
|
465
|
+
urStatus: urIngestSessionFsPathEntry.urStatus,
|
|
466
|
+
urDiagnostics: urIngestSessionFsPathEntry.urDiagnostics
|
|
467
|
+
}).from(urIngestSessionFsPathEntry).innerJoin(urIngestSessionFsPath, eq(urIngestSessionFsPathEntry.ingestFsPathId, urIngestSessionFsPath.urIngestSessionFsPathId)).innerJoin(urIngestSession, eq(urIngestSessionFsPath.ingestSessionId, urIngestSession.urIngestSessionId)).where(sql`${urIngestSessionFsPathEntry.urStatus} IS NOT NULL`).groupBy(
|
|
468
|
+
urIngestSession.deviceId,
|
|
469
|
+
urIngestSession.urIngestSessionId,
|
|
470
|
+
urIngestSessionFsPath.urIngestSessionFsPathId,
|
|
471
|
+
urIngestSessionFsPath.rootPath,
|
|
472
|
+
urIngestSessionFsPathEntry.urIngestSessionFsPathEntryId,
|
|
473
|
+
urIngestSessionFsPathEntry.filePathAbs,
|
|
474
|
+
urIngestSessionFsPathEntry.urStatus,
|
|
475
|
+
urIngestSessionFsPathEntry.urDiagnostics
|
|
476
|
+
);
|
|
477
|
+
});
|
|
478
|
+
var orchestrationSessionByDevice = sqliteView("orchestration_session_by_device").as((qb) => {
|
|
479
|
+
return qb.select({
|
|
480
|
+
deviceId: device.deviceId,
|
|
481
|
+
deviceName: device.name,
|
|
482
|
+
sessionCount: count(orchestrationSession.orchestrationSessionId).as("session_count")
|
|
483
|
+
}).from(orchestrationSession).innerJoin(device, eq(orchestrationSession.deviceId, device.deviceId)).groupBy(device.deviceId, device.name);
|
|
484
|
+
});
|
|
485
|
+
var orchestrationSessionDuration = sqliteView("orchestration_session_duration").as((qb) => {
|
|
486
|
+
return qb.select({
|
|
487
|
+
orchestrationSessionId: orchestrationSession.orchestrationSessionId,
|
|
488
|
+
orchestrationNature: orchestrationNature.nature,
|
|
489
|
+
orchStartedAt: orchestrationSession.orchStartedAt,
|
|
490
|
+
orchFinishedAt: orchestrationSession.orchFinishedAt,
|
|
491
|
+
durationSeconds: sql`(JULIANDAY(${orchestrationSession.orchFinishedAt}) - JULIANDAY(${orchestrationSession.orchStartedAt})) * 24 * 60 * 60`.as("duration_seconds")
|
|
492
|
+
}).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).where(sql`${orchestrationSession.orchFinishedAt} IS NOT NULL`);
|
|
493
|
+
});
|
|
494
|
+
var orchestrationSuccessRate = sqliteView("orchestration_success_rate").as((qb) => {
|
|
495
|
+
return qb.select({
|
|
496
|
+
orchestrationNature: orchestrationNature.nature,
|
|
497
|
+
totalSessions: count().as("total_sessions"),
|
|
498
|
+
successfulSessions: sum(sql`CASE WHEN ${orchestrationSessionState.toState} = 'surveilr_orch_completed' THEN 1 ELSE 0 END`).as("successful_sessions"),
|
|
499
|
+
successRate: sql`(CAST(SUM(CASE WHEN ${orchestrationSessionState.toState} = 'surveilr_orch_completed' THEN 1 ELSE 0 END) AS FLOAT) / COUNT(*)) * 100`.as("success_rate")
|
|
500
|
+
}).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionState, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionState.sessionId)).where(sql`${orchestrationSessionState.toState} IN ('surveilr_orch_completed', 'surveilr_orch_failed')`).groupBy(orchestrationNature.nature);
|
|
501
|
+
});
|
|
502
|
+
var orchestrationSessionScript = sqliteView("orchestration_session_script").as((qb) => {
|
|
503
|
+
return qb.select({
|
|
504
|
+
orchestrationSessionId: orchestrationSession.orchestrationSessionId,
|
|
505
|
+
orchestrationNature: orchestrationNature.nature,
|
|
506
|
+
scriptCount: count(orchestrationSessionEntry.orchestrationSessionEntryId).as("script_count")
|
|
507
|
+
}).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionEntry, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionEntry.sessionId)).groupBy(orchestrationSession.orchestrationSessionId, orchestrationNature.nature);
|
|
508
|
+
});
|
|
509
|
+
var orchestrationExecutionsByType = sqliteView("orchestration_executions_by_type").as((qb) => {
|
|
510
|
+
return qb.select({
|
|
511
|
+
execNature: orchestrationSessionExec.execNature,
|
|
512
|
+
executionCount: count(orchestrationSessionExec.orchestrationSessionExecId).as("execution_count")
|
|
513
|
+
}).from(orchestrationSessionExec).groupBy(orchestrationSessionExec.execNature);
|
|
514
|
+
});
|
|
515
|
+
var orchestrationExecutionSuccessRateByType = sqliteView("orchestration_execution_success_rate_by_type").as((qb) => {
|
|
516
|
+
return qb.select({
|
|
517
|
+
execNature: orchestrationSessionExec.execNature,
|
|
518
|
+
totalExecutions: count().as("total_executions"),
|
|
519
|
+
successfulExecutions: sum(sql`CASE WHEN ${orchestrationSessionExec.execStatus} = 0 THEN 1 ELSE 0 END`).as("successful_executions"),
|
|
520
|
+
successRate: sql`(CAST(SUM(CASE WHEN ${orchestrationSessionExec.execStatus} = 0 THEN 1 ELSE 0 END) AS FLOAT) / COUNT(*)) * 100`.as("success_rate")
|
|
521
|
+
}).from(orchestrationSessionExec).groupBy(orchestrationSessionExec.execNature);
|
|
522
|
+
});
|
|
523
|
+
var orchestrationSessionSummary = sqliteView("orchestration_session_summary").as((qb) => {
|
|
524
|
+
return qb.select({
|
|
525
|
+
issueType: orchestrationSessionIssue.issueType,
|
|
526
|
+
issueCount: count(orchestrationSessionIssue.orchestrationSessionIssueId).as("issue_count")
|
|
527
|
+
}).from(orchestrationSessionIssue).groupBy(orchestrationSessionIssue.issueType);
|
|
528
|
+
});
|
|
529
|
+
var orchestrationIssueRemediation = sqliteView("orchestration_issue_remediation").as((qb) => {
|
|
530
|
+
return qb.select({
|
|
531
|
+
orchestrationSessionIssueId: orchestrationSessionIssue.orchestrationSessionIssueId,
|
|
532
|
+
issueType: orchestrationSessionIssue.issueType,
|
|
533
|
+
issueMessage: orchestrationSessionIssue.issueMessage,
|
|
534
|
+
remediation: orchestrationSessionIssue.remediation
|
|
535
|
+
}).from(orchestrationSessionIssue).where(sql`${orchestrationSessionIssue.remediation} IS NOT NULL`);
|
|
536
|
+
});
|
|
537
|
+
var orchestrationLogsBySession = sqliteView("orchestration_logs_by_session").as((qb) => {
|
|
538
|
+
return qb.select({
|
|
539
|
+
orchestrationSessionId: orchestrationSession.orchestrationSessionId,
|
|
540
|
+
orchestrationNature: orchestrationNature.nature,
|
|
541
|
+
category: orchestrationSessionLog.category,
|
|
542
|
+
logCount: count(orchestrationSessionLog.orchestrationSessionLogId).as("log_count")
|
|
543
|
+
}).from(orchestrationSession).innerJoin(orchestrationNature, eq(orchestrationSession.orchestrationNatureId, orchestrationNature.orchestrationNatureId)).innerJoin(orchestrationSessionExec, eq(orchestrationSession.orchestrationSessionId, orchestrationSessionExec.sessionId)).innerJoin(orchestrationSessionLog, eq(orchestrationSessionExec.orchestrationSessionExecId, orchestrationSessionLog.parentExecId)).groupBy(orchestrationSession.orchestrationSessionId, orchestrationNature.nature, orchestrationSessionLog.category);
|
|
544
|
+
});
|
|
545
|
+
var emailMessagesWithTimezone = sqliteView("email_messages_with_timezone", {}).as(sql`
|
|
546
|
+
SELECT
|
|
547
|
+
m.*,
|
|
548
|
+
-- Smart timezone conversion based on original timezone
|
|
549
|
+
CASE
|
|
550
|
+
-- If already IST (+0530), just extract the time
|
|
551
|
+
WHEN date LIKE '%+0530%' THEN substr(date, 18, 8)
|
|
552
|
+
-- If UTC (+0000), convert to IST
|
|
553
|
+
WHEN date LIKE '%+0000%' THEN time(substr(date, 18, 8), '+5 hours', '+30 minutes')
|
|
554
|
+
-- If EST (-0500), convert to IST
|
|
555
|
+
WHEN date LIKE '%-0500%' THEN time(substr(date, 18, 8), '+10 hours', '+30 minutes')
|
|
556
|
+
-- If PST (-0800), convert to IST
|
|
557
|
+
WHEN date LIKE '%-0800%' THEN time(substr(date, 18, 8), '+13 hours', '+30 minutes')
|
|
558
|
+
-- Default: assume UTC and convert
|
|
559
|
+
ELSE time(substr(date, 18, 8), '+5 hours', '+30 minutes')
|
|
560
|
+
END as ist_time,
|
|
561
|
+
|
|
562
|
+
-- Always show UTC time (convert from any timezone to UTC first)
|
|
563
|
+
CASE
|
|
564
|
+
WHEN date LIKE '%+0530%' THEN time(substr(date, 18, 8), '-5 hours', '-30 minutes')
|
|
565
|
+
WHEN date LIKE '%+0000%' THEN substr(date, 18, 8)
|
|
566
|
+
WHEN date LIKE '%-0500%' THEN time(substr(date, 18, 8), '+5 hours')
|
|
567
|
+
WHEN date LIKE '%-0800%' THEN time(substr(date, 18, 8), '+8 hours')
|
|
568
|
+
ELSE substr(date, 18, 8)
|
|
569
|
+
END as utc_time,
|
|
570
|
+
|
|
571
|
+
-- Detect original timezone more accurately
|
|
572
|
+
CASE
|
|
573
|
+
WHEN date LIKE '%+0530%' THEN 'IST'
|
|
574
|
+
WHEN date LIKE '%+0000%' THEN 'UTC'
|
|
575
|
+
WHEN date LIKE '%-0500%' THEN 'EST'
|
|
576
|
+
WHEN date LIKE '%-0800%' THEN 'PST'
|
|
577
|
+
ELSE 'Unknown'
|
|
578
|
+
END as detected_timezone
|
|
579
|
+
FROM ur_ingest_session_imap_acct_folder_message m`);
|
|
580
|
+
var deviceGraphOverview = sqliteView("device_graph_overview", {}).as(sql`
|
|
581
|
+
SELECT
|
|
582
|
+
'snmp_device' as device_type,
|
|
583
|
+
sd.device_key as device_id,
|
|
584
|
+
sd.snmp_host as primary_identifier,
|
|
585
|
+
sd.device_type as device_category,
|
|
586
|
+
sd.device_description as description,
|
|
587
|
+
sd.status,
|
|
588
|
+
sd.created_at,
|
|
589
|
+
sd.updated_at,
|
|
590
|
+
json_object(
|
|
591
|
+
'snmp_port', sd.snmp_port,
|
|
592
|
+
'snmp_community', sd.snmp_community,
|
|
593
|
+
'snmp_version', sd.snmp_version,
|
|
594
|
+
'snmp_timeout', sd.snmp_timeout,
|
|
595
|
+
'snmp_retries', sd.snmp_retries
|
|
596
|
+
) as device_metadata
|
|
597
|
+
FROM surveilr_snmp_device sd
|
|
598
|
+
WHERE sd.status = 'active'
|
|
599
|
+
|
|
600
|
+
UNION ALL
|
|
601
|
+
|
|
602
|
+
SELECT
|
|
603
|
+
'osquery_node' as device_type,
|
|
604
|
+
osq.node_key as device_id,
|
|
605
|
+
osq.host_identifier as primary_identifier,
|
|
606
|
+
'endpoint' as device_category,
|
|
607
|
+
osq.os_version as description,
|
|
608
|
+
osq.status,
|
|
609
|
+
osq.created_at,
|
|
610
|
+
osq.updated_at,
|
|
611
|
+
json_object(
|
|
612
|
+
'last_seen', osq.last_seen,
|
|
613
|
+
'platform', osq.platform,
|
|
614
|
+
'osquery_version', osq.osquery_version,
|
|
615
|
+
'device_id', osq.device_id
|
|
616
|
+
) as device_metadata
|
|
617
|
+
FROM surveilr_osquery_ms_node osq
|
|
618
|
+
WHERE osq.status = 'active'`);
|
|
619
|
+
var deviceGraphCorrelations = sqliteView("device_graph_correlations", {}).as(sql`
|
|
620
|
+
SELECT DISTINCT
|
|
621
|
+
ure1.node_id as snmp_device_key,
|
|
622
|
+
ure1.uniform_resource_id as osquery_node_key,
|
|
623
|
+
CASE
|
|
624
|
+
WHEN ure1.nature LIKE '%ip_address_match%' THEN 'IP Address Match'
|
|
625
|
+
WHEN ure1.nature LIKE '%hostname_match%' THEN 'Hostname Match'
|
|
626
|
+
WHEN ure1.nature LIKE '%mac_address_match%' THEN 'MAC Address Match'
|
|
627
|
+
WHEN ure1.nature LIKE '%network_topology%' THEN 'Network Topology'
|
|
628
|
+
WHEN ure1.nature LIKE '%manual%' THEN 'Manual'
|
|
629
|
+
ELSE 'Unknown'
|
|
630
|
+
END as correlation_type,
|
|
631
|
+
CASE
|
|
632
|
+
WHEN ure1.nature LIKE '%ip_address_match%' THEN 0.95
|
|
633
|
+
WHEN ure1.nature LIKE '%hostname_match%' THEN 0.85
|
|
634
|
+
WHEN ure1.nature LIKE '%mac_address_match%' THEN 0.90
|
|
635
|
+
WHEN ure1.nature LIKE '%network_topology%' THEN 0.70
|
|
636
|
+
WHEN ure1.nature LIKE '%manual%' THEN 1.0
|
|
637
|
+
ELSE 0.5
|
|
638
|
+
END as confidence_score,
|
|
639
|
+
json_extract(ure1.elaboration, '$.snmp_host') as snmp_host,
|
|
640
|
+
json_extract(ure1.elaboration, '$.correlation_method') as correlation_method,
|
|
641
|
+
ure1.elaboration as evidence
|
|
642
|
+
FROM uniform_resource_edge ure1
|
|
643
|
+
WHERE ure1.graph_name = 'device_graph'
|
|
644
|
+
AND ure1.nature LIKE 'snmp_to_osquery_%'
|
|
645
|
+
ORDER BY ure1.node_id, ure1.uniform_resource_id`);
|
|
646
|
+
var snmpDeviceInventory = sqliteView("snmp_device_inventory", {}).as(sql`
|
|
647
|
+
SELECT
|
|
648
|
+
sd.device_key,
|
|
649
|
+
sd.snmp_host,
|
|
650
|
+
sd.snmp_port,
|
|
651
|
+
sd.device_type,
|
|
652
|
+
sd.device_description,
|
|
653
|
+
sd.status,
|
|
654
|
+
sd.created_at,
|
|
655
|
+
COUNT(sc.surveilr_snmp_collection_id) as total_collections,
|
|
656
|
+
MAX(sc.collected_at) as last_collection_time,
|
|
657
|
+
COUNT(DISTINCT sc.oid) as unique_oids_collected,
|
|
658
|
+
AVG(
|
|
659
|
+
CASE
|
|
660
|
+
WHEN sc.collected_at > datetime('now', '-1 hour') THEN 1
|
|
661
|
+
ELSE 0
|
|
662
|
+
END
|
|
663
|
+
) as recent_activity_score
|
|
664
|
+
FROM surveilr_snmp_device sd
|
|
665
|
+
LEFT JOIN surveilr_snmp_collection sc ON sd.device_key = sc.device_key
|
|
666
|
+
GROUP BY sd.device_key, sd.snmp_host, sd.snmp_port, sd.device_type,
|
|
667
|
+
sd.device_description, sd.status, sd.created_at
|
|
668
|
+
ORDER BY sd.status, sd.snmp_host`);
|
|
669
|
+
var snmpCollectionSummary = sqliteView("snmp_collection_summary", {}).as(sql`
|
|
670
|
+
SELECT
|
|
671
|
+
sc.device_key,
|
|
672
|
+
sd.snmp_host,
|
|
673
|
+
sc.oid_type,
|
|
674
|
+
COUNT(*) as collection_count,
|
|
675
|
+
COUNT(DISTINCT sc.oid) as unique_oids,
|
|
676
|
+
MIN(sc.collected_at) as first_collection,
|
|
677
|
+
MAX(sc.collected_at) as latest_collection,
|
|
678
|
+
COUNT(DISTINCT DATE(sc.collected_at)) as collection_days
|
|
679
|
+
FROM surveilr_snmp_collection sc
|
|
680
|
+
JOIN surveilr_snmp_device sd ON sc.device_key = sd.device_key
|
|
681
|
+
GROUP BY sc.device_key, sd.snmp_host, sc.oid_type
|
|
682
|
+
ORDER BY sd.snmp_host, sc.oid_type`);
|
|
683
|
+
var networkDeviceTopology = sqliteView("network_device_topology", {}).as(sql`
|
|
684
|
+
SELECT
|
|
685
|
+
sd.device_key,
|
|
686
|
+
sd.snmp_host,
|
|
687
|
+
sd.device_type,
|
|
688
|
+
-- Extract network information from SNMP collections
|
|
689
|
+
COALESCE(
|
|
690
|
+
(SELECT sc.oid_value
|
|
691
|
+
FROM surveilr_snmp_collection sc
|
|
692
|
+
WHERE sc.device_key = sd.device_key
|
|
693
|
+
AND sc.oid LIKE '%.1.3.6.1.2.1.1.1.%'
|
|
694
|
+
LIMIT 1),
|
|
695
|
+
'Unknown'
|
|
696
|
+
) as system_description,
|
|
697
|
+
COALESCE(
|
|
698
|
+
(SELECT sc.oid_value
|
|
699
|
+
FROM surveilr_snmp_collection sc
|
|
700
|
+
WHERE sc.device_key = sd.device_key
|
|
701
|
+
AND sc.oid LIKE '%.1.3.6.1.2.1.1.5.%'
|
|
702
|
+
LIMIT 1),
|
|
703
|
+
'Unknown'
|
|
704
|
+
) as system_name,
|
|
705
|
+
-- Count correlated devices
|
|
706
|
+
(SELECT COUNT(*)
|
|
707
|
+
FROM device_graph_correlations dgc
|
|
708
|
+
WHERE dgc.snmp_device_key = sd.device_key
|
|
709
|
+
) as correlated_devices_count,
|
|
710
|
+
sd.status,
|
|
711
|
+
sd.created_at
|
|
712
|
+
FROM surveilr_snmp_device sd
|
|
713
|
+
WHERE sd.status = 'active'
|
|
714
|
+
ORDER BY sd.snmp_host`);
|
|
715
|
+
var deviceGraphAnalytics = sqliteView("device_graph_analytics", {}).as(sql`
|
|
716
|
+
SELECT
|
|
717
|
+
'summary' as metric_type,
|
|
718
|
+
'total_snmp_devices' as metric_name,
|
|
719
|
+
COUNT(*) as metric_value,
|
|
720
|
+
NULL as additional_info
|
|
721
|
+
FROM surveilr_snmp_device
|
|
722
|
+
WHERE status = 'active'
|
|
723
|
+
|
|
724
|
+
UNION ALL
|
|
725
|
+
|
|
726
|
+
SELECT
|
|
727
|
+
'summary' as metric_type,
|
|
728
|
+
'total_osquery_nodes' as metric_name,
|
|
729
|
+
COUNT(*) as metric_value,
|
|
730
|
+
NULL as additional_info
|
|
731
|
+
FROM surveilr_osquery_ms_node
|
|
732
|
+
WHERE status = 'active'
|
|
733
|
+
|
|
734
|
+
UNION ALL
|
|
735
|
+
|
|
736
|
+
SELECT
|
|
737
|
+
'summary' as metric_type,
|
|
738
|
+
'total_correlations' as metric_name,
|
|
739
|
+
COUNT(*) as metric_value,
|
|
740
|
+
NULL as additional_info
|
|
741
|
+
FROM device_graph_correlations
|
|
742
|
+
|
|
743
|
+
UNION ALL
|
|
744
|
+
|
|
745
|
+
SELECT
|
|
746
|
+
'summary' as metric_type,
|
|
747
|
+
'correlation_types' as metric_name,
|
|
748
|
+
COUNT(DISTINCT correlation_type) as metric_value,
|
|
749
|
+
GROUP_CONCAT(DISTINCT correlation_type) as additional_info
|
|
750
|
+
FROM device_graph_correlations
|
|
751
|
+
|
|
752
|
+
UNION ALL
|
|
753
|
+
|
|
754
|
+
SELECT
|
|
755
|
+
'correlation_by_type' as metric_type,
|
|
756
|
+
correlation_type as metric_name,
|
|
757
|
+
COUNT(*) as metric_value,
|
|
758
|
+
printf('avg_confidence: %.2f', AVG(confidence_score)) as additional_info
|
|
759
|
+
FROM device_graph_correlations
|
|
760
|
+
GROUP BY correlation_type
|
|
761
|
+
|
|
762
|
+
UNION ALL
|
|
763
|
+
|
|
764
|
+
SELECT
|
|
765
|
+
'collection_stats' as metric_type,
|
|
766
|
+
'total_snmp_collections' as metric_name,
|
|
767
|
+
COUNT(*) as metric_value,
|
|
768
|
+
printf('unique_devices: %d', COUNT(DISTINCT device_key)) as additional_info
|
|
769
|
+
FROM surveilr_snmp_collection
|
|
770
|
+
|
|
771
|
+
UNION ALL
|
|
772
|
+
|
|
773
|
+
SELECT
|
|
774
|
+
'collection_stats' as metric_type,
|
|
775
|
+
'collection_activity_last_24h' as metric_name,
|
|
776
|
+
COUNT(*) as metric_value,
|
|
777
|
+
printf('devices_active: %d', COUNT(DISTINCT device_key)) as additional_info
|
|
778
|
+
FROM surveilr_snmp_collection
|
|
779
|
+
WHERE collected_at > datetime('now', '-24 hours')`);
|
|
780
|
+
var snmpOidPerformance = sqliteView("snmp_oid_performance", {}).as(sql`
|
|
781
|
+
SELECT
|
|
782
|
+
sc.oid,
|
|
783
|
+
sc.oid_type,
|
|
784
|
+
COUNT(*) as total_collections,
|
|
785
|
+
COUNT(DISTINCT sc.device_key) as devices_collecting,
|
|
786
|
+
COUNT(DISTINCT DATE(sc.collected_at)) as collection_days,
|
|
787
|
+
MIN(sc.collected_at) as first_seen,
|
|
788
|
+
MAX(sc.collected_at) as last_seen,
|
|
789
|
+
-- Calculate collection frequency (avoid nested aggregates)
|
|
790
|
+
CAST(
|
|
791
|
+
COUNT(*) * 1.0 /
|
|
792
|
+
NULLIF(MAX(julianday(sc.collected_at)) - MIN(julianday(sc.collected_at)), 0)
|
|
793
|
+
AS REAL) as avg_collections_per_day,
|
|
794
|
+
-- Sample values for diversity analysis
|
|
795
|
+
COUNT(DISTINCT sc.oid_value) as unique_values_count,
|
|
796
|
+
GROUP_CONCAT(DISTINCT sc.oid_value) as sample_values
|
|
797
|
+
FROM surveilr_snmp_collection sc
|
|
798
|
+
GROUP BY sc.oid, sc.oid_type
|
|
799
|
+
HAVING COUNT(*) > 0
|
|
800
|
+
ORDER BY total_collections DESC, devices_collecting DESC`);
|
|
801
|
+
|
|
802
|
+
export {
|
|
803
|
+
codeNotebookCellVersions,
|
|
804
|
+
codeNotebookCellLatest,
|
|
805
|
+
codeNotebookSqlCellMigratableVersion,
|
|
806
|
+
codeNotebookSqlCellMigratable,
|
|
807
|
+
codeNotebookSqlCellMigratableState,
|
|
808
|
+
codeNotebookSqlCellMigratableNotExecuted,
|
|
809
|
+
codeNotebookMigrationSql,
|
|
810
|
+
imapGraph,
|
|
811
|
+
filesystemGraph,
|
|
812
|
+
consoleInformationSchemaTable,
|
|
813
|
+
consoleInformationSchemaView,
|
|
814
|
+
consoleContentTabular,
|
|
815
|
+
consoleInformationSchemaTableColFkey,
|
|
816
|
+
consoleInformationSchemaTableColIndex,
|
|
817
|
+
rssdStatisticsOverview,
|
|
818
|
+
rssdTableStatistic,
|
|
819
|
+
uniformResourceFile,
|
|
820
|
+
uniformResourceImap,
|
|
821
|
+
uniformResourceImapContent,
|
|
822
|
+
urIngestSessionFilesStats,
|
|
823
|
+
urIngestSessionFilesStatsLatest,
|
|
824
|
+
urIngestSessionTasksStats,
|
|
825
|
+
urIngestSessionTasksStatsLatest,
|
|
826
|
+
urIngestSessionFileIssue,
|
|
827
|
+
orchestrationSessionByDevice,
|
|
828
|
+
orchestrationSessionDuration,
|
|
829
|
+
orchestrationSuccessRate,
|
|
830
|
+
orchestrationSessionScript,
|
|
831
|
+
orchestrationExecutionsByType,
|
|
832
|
+
orchestrationExecutionSuccessRateByType,
|
|
833
|
+
orchestrationSessionSummary,
|
|
834
|
+
orchestrationIssueRemediation,
|
|
835
|
+
orchestrationLogsBySession,
|
|
836
|
+
emailMessagesWithTimezone,
|
|
837
|
+
deviceGraphOverview,
|
|
838
|
+
deviceGraphCorrelations,
|
|
839
|
+
snmpDeviceInventory,
|
|
840
|
+
snmpCollectionSummary,
|
|
841
|
+
networkDeviceTopology,
|
|
842
|
+
deviceGraphAnalytics,
|
|
843
|
+
snmpOidPerformance
|
|
844
|
+
};
|