@hailer/mcp 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/.claude/commands/tool-builder.md +37 -0
  2. package/.claude/commands/ws-pull.md +44 -0
  3. package/.claude/settings.json +8 -0
  4. package/.claude/settings.local.json +49 -0
  5. package/.claude/skills/activity-api/SKILL.md +96 -0
  6. package/.claude/skills/activity-api/references/activity-endpoints.md +845 -0
  7. package/.claude/skills/add-app-member-skill/SKILL.md +977 -0
  8. package/.claude/skills/agent-building/SKILL.md +243 -0
  9. package/.claude/skills/agent-building/references/architecture-patterns.md +446 -0
  10. package/.claude/skills/agent-building/references/code-examples.md +587 -0
  11. package/.claude/skills/agent-building/references/implementation-guide.md +619 -0
  12. package/.claude/skills/app-api/SKILL.md +219 -0
  13. package/.claude/skills/app-api/references/app-endpoints.md +759 -0
  14. package/.claude/skills/building-hailer-apps-skill/SKILL.md +548 -0
  15. package/.claude/skills/create-app-skill/SKILL.md +1101 -0
  16. package/.claude/skills/create-insight-skill/SKILL.md +1317 -0
  17. package/.claude/skills/get-insight-data-skill/SKILL.md +1053 -0
  18. package/.claude/skills/hailer-api/SKILL.md +283 -0
  19. package/.claude/skills/hailer-api/references/activities.md +620 -0
  20. package/.claude/skills/hailer-api/references/authentication.md +216 -0
  21. package/.claude/skills/hailer-api/references/datasets.md +437 -0
  22. package/.claude/skills/hailer-api/references/files.md +301 -0
  23. package/.claude/skills/hailer-api/references/insights.md +469 -0
  24. package/.claude/skills/hailer-api/references/workflows.md +720 -0
  25. package/.claude/skills/hailer-api/references/workspaces-users.md +445 -0
  26. package/.claude/skills/insight-api/SKILL.md +185 -0
  27. package/.claude/skills/insight-api/references/insight-endpoints.md +514 -0
  28. package/.claude/skills/install-workflow-skill/SKILL.md +1056 -0
  29. package/.claude/skills/list-apps-skill/SKILL.md +1010 -0
  30. package/.claude/skills/list-workflows-minimal-skill/SKILL.md +992 -0
  31. package/.claude/skills/local-first-skill/SKILL.md +570 -0
  32. package/.claude/skills/mcp-tools/SKILL.md +419 -0
  33. package/.claude/skills/mcp-tools/references/api-endpoints.md +499 -0
  34. package/.claude/skills/mcp-tools/references/data-structures.md +554 -0
  35. package/.claude/skills/mcp-tools/references/implementation-patterns.md +717 -0
  36. package/.claude/skills/preview-insight-skill/SKILL.md +1290 -0
  37. package/.claude/skills/publish-hailer-app-skill/SKILL.md +453 -0
  38. package/.claude/skills/remove-app-member-skill/SKILL.md +671 -0
  39. package/.claude/skills/remove-app-skill/SKILL.md +985 -0
  40. package/.claude/skills/remove-insight-skill/SKILL.md +1011 -0
  41. package/.claude/skills/remove-workflow-skill/SKILL.md +920 -0
  42. package/.claude/skills/scaffold-hailer-app-skill/SKILL.md +1034 -0
  43. package/.claude/skills/skill-testing/README.md +137 -0
  44. package/.claude/skills/skill-testing/SKILL.md +348 -0
  45. package/.claude/skills/skill-testing/references/test-patterns.md +705 -0
  46. package/.claude/skills/skill-testing/references/testing-guide.md +603 -0
  47. package/.claude/skills/skill-testing/references/validation-checklist.md +537 -0
  48. package/.claude/skills/tool-builder/SKILL.md +328 -0
  49. package/.claude/skills/update-app-skill/SKILL.md +970 -0
  50. package/.claude/skills/update-workflow-field-skill/SKILL.md +1098 -0
  51. package/.env.example +81 -0
  52. package/.mcp.json +13 -0
  53. package/README.md +297 -0
  54. package/dist/app.d.ts +4 -0
  55. package/dist/app.js +74 -0
  56. package/dist/cli.d.ts +3 -0
  57. package/dist/cli.js +5 -0
  58. package/dist/client/adaptive-documentation-bot.d.ts +108 -0
  59. package/dist/client/adaptive-documentation-bot.js +475 -0
  60. package/dist/client/adaptive-documentation-types.d.ts +66 -0
  61. package/dist/client/adaptive-documentation-types.js +9 -0
  62. package/dist/client/agent-activity-bot.d.ts +51 -0
  63. package/dist/client/agent-activity-bot.js +166 -0
  64. package/dist/client/agent-tracker.d.ts +499 -0
  65. package/dist/client/agent-tracker.js +659 -0
  66. package/dist/client/description-updater.d.ts +56 -0
  67. package/dist/client/description-updater.js +259 -0
  68. package/dist/client/log-parser.d.ts +72 -0
  69. package/dist/client/log-parser.js +387 -0
  70. package/dist/client/mcp-client.d.ts +50 -0
  71. package/dist/client/mcp-client.js +532 -0
  72. package/dist/client/message-processor.d.ts +35 -0
  73. package/dist/client/message-processor.js +352 -0
  74. package/dist/client/multi-bot-manager.d.ts +24 -0
  75. package/dist/client/multi-bot-manager.js +74 -0
  76. package/dist/client/providers/anthropic-provider.d.ts +19 -0
  77. package/dist/client/providers/anthropic-provider.js +631 -0
  78. package/dist/client/providers/llm-provider.d.ts +47 -0
  79. package/dist/client/providers/llm-provider.js +367 -0
  80. package/dist/client/providers/openai-provider.d.ts +23 -0
  81. package/dist/client/providers/openai-provider.js +621 -0
  82. package/dist/client/simple-llm-caller.d.ts +19 -0
  83. package/dist/client/simple-llm-caller.js +100 -0
  84. package/dist/client/skill-generator.d.ts +81 -0
  85. package/dist/client/skill-generator.js +386 -0
  86. package/dist/client/test-adaptive-bot.d.ts +9 -0
  87. package/dist/client/test-adaptive-bot.js +82 -0
  88. package/dist/client/token-pricing.d.ts +38 -0
  89. package/dist/client/token-pricing.js +127 -0
  90. package/dist/client/token-tracker.d.ts +232 -0
  91. package/dist/client/token-tracker.js +457 -0
  92. package/dist/client/token-usage-bot.d.ts +53 -0
  93. package/dist/client/token-usage-bot.js +153 -0
  94. package/dist/client/tool-executor.d.ts +69 -0
  95. package/dist/client/tool-executor.js +159 -0
  96. package/dist/client/tool-schema-loader.d.ts +60 -0
  97. package/dist/client/tool-schema-loader.js +178 -0
  98. package/dist/client/types.d.ts +69 -0
  99. package/dist/client/types.js +7 -0
  100. package/dist/config.d.ts +162 -0
  101. package/dist/config.js +296 -0
  102. package/dist/core.d.ts +26 -0
  103. package/dist/core.js +147 -0
  104. package/dist/lib/context-manager.d.ts +111 -0
  105. package/dist/lib/context-manager.js +431 -0
  106. package/dist/lib/logger.d.ts +74 -0
  107. package/dist/lib/logger.js +277 -0
  108. package/dist/lib/materialize.d.ts +3 -0
  109. package/dist/lib/materialize.js +101 -0
  110. package/dist/lib/normalizedName.d.ts +7 -0
  111. package/dist/lib/normalizedName.js +48 -0
  112. package/dist/lib/prompt-length-manager.d.ts +81 -0
  113. package/dist/lib/prompt-length-manager.js +457 -0
  114. package/dist/lib/terminal-prompt.d.ts +9 -0
  115. package/dist/lib/terminal-prompt.js +108 -0
  116. package/dist/mcp/UserContextCache.d.ts +56 -0
  117. package/dist/mcp/UserContextCache.js +163 -0
  118. package/dist/mcp/auth.d.ts +2 -0
  119. package/dist/mcp/auth.js +29 -0
  120. package/dist/mcp/hailer-clients.d.ts +42 -0
  121. package/dist/mcp/hailer-clients.js +246 -0
  122. package/dist/mcp/signal-handler.d.ts +45 -0
  123. package/dist/mcp/signal-handler.js +317 -0
  124. package/dist/mcp/tool-registry.d.ts +100 -0
  125. package/dist/mcp/tool-registry.js +306 -0
  126. package/dist/mcp/tools/activity.d.ts +15 -0
  127. package/dist/mcp/tools/activity.js +955 -0
  128. package/dist/mcp/tools/app.d.ts +20 -0
  129. package/dist/mcp/tools/app.js +1488 -0
  130. package/dist/mcp/tools/discussion.d.ts +19 -0
  131. package/dist/mcp/tools/discussion.js +950 -0
  132. package/dist/mcp/tools/file.d.ts +15 -0
  133. package/dist/mcp/tools/file.js +119 -0
  134. package/dist/mcp/tools/insight.d.ts +17 -0
  135. package/dist/mcp/tools/insight.js +806 -0
  136. package/dist/mcp/tools/skill.d.ts +10 -0
  137. package/dist/mcp/tools/skill.js +279 -0
  138. package/dist/mcp/tools/user.d.ts +10 -0
  139. package/dist/mcp/tools/user.js +108 -0
  140. package/dist/mcp/tools/workflow-template.d.ts +19 -0
  141. package/dist/mcp/tools/workflow-template.js +822 -0
  142. package/dist/mcp/tools/workflow.d.ts +18 -0
  143. package/dist/mcp/tools/workflow.js +1362 -0
  144. package/dist/mcp/utils/api-errors.d.ts +45 -0
  145. package/dist/mcp/utils/api-errors.js +160 -0
  146. package/dist/mcp/utils/data-transformers.d.ts +102 -0
  147. package/dist/mcp/utils/data-transformers.js +194 -0
  148. package/dist/mcp/utils/file-upload.d.ts +33 -0
  149. package/dist/mcp/utils/file-upload.js +148 -0
  150. package/dist/mcp/utils/hailer-api-client.d.ts +120 -0
  151. package/dist/mcp/utils/hailer-api-client.js +323 -0
  152. package/dist/mcp/utils/index.d.ts +13 -0
  153. package/dist/mcp/utils/index.js +39 -0
  154. package/dist/mcp/utils/logger.d.ts +42 -0
  155. package/dist/mcp/utils/logger.js +103 -0
  156. package/dist/mcp/utils/types.d.ts +286 -0
  157. package/dist/mcp/utils/types.js +7 -0
  158. package/dist/mcp/workspace-cache.d.ts +42 -0
  159. package/dist/mcp/workspace-cache.js +97 -0
  160. package/dist/mcp-server.d.ts +42 -0
  161. package/dist/mcp-server.js +280 -0
  162. package/package.json +56 -0
  163. package/tsconfig.json +23 -0
@@ -0,0 +1,1053 @@
1
+ ---
2
+ name: Getting Insight Data
3
+ description: Complete guide for executing insights and retrieving results - use when running reports or exporting data
4
+ ---
5
+
6
+ # Getting Insight Data - Complete Guide
7
+
8
+ Complete reference for executing Hailer insights and retrieving full results using the `get_insight_data` MCP tool.
9
+
10
+ ## Table of Contents
11
+ 1. [Quick Reference](#quick-reference)
12
+ 2. [Overview](#overview)
13
+ 3. [Core Concepts](#core-concepts)
14
+ 4. [Basic Usage](#basic-usage)
15
+ 5. [Update vs Cached Data](#update-vs-cached-data)
16
+ 6. [Working with Results](#working-with-results)
17
+ 7. [Large Datasets](#large-datasets)
18
+ 8. [Common Scenarios](#common-scenarios)
19
+ 9. [Performance Considerations](#performance-considerations)
20
+ 10. [Best Practices](#best-practices)
21
+ 11. [Troubleshooting](#troubleshooting)
22
+
23
+ ## Quick Reference
24
+
25
+ **Get Cached Results (Fast):**
26
+
27
+ ```javascript
28
+ get_insight_data({
29
+ insightId: 'insight-id'
30
+ })
31
+ ```
32
+
33
+ **Force Refresh (Recalculate):**
34
+
35
+ ```javascript
36
+ get_insight_data({
37
+ insightId: 'insight-id',
38
+ update: true // Recalculate from current data
39
+ })
40
+ ```
41
+
42
+ **Response Format:**
43
+ ```javascript
44
+ {
45
+ columns: ['title', 'priority', 'dueDate'],
46
+ rows: [
47
+ ['Task 1', 'High', '2024-12-31'],
48
+ ['Task 2', 'Medium', '2024-12-25'],
49
+ // ... all matching activities
50
+ ]
51
+ }
52
+ ```
53
+
54
+ **Key Features:**
55
+ - Executes saved insight SQL query
56
+ - Returns ALL matching activities (not limited like preview)
57
+ - Supports cached results (fast) or fresh calculation (slower)
58
+ - Data returned in tabular format (columns + rows)
59
+ - Can handle large result sets
60
+
61
+ ## Overview
62
+
63
+ **What is get_insight_data?**
64
+
65
+ This tool executes a saved insight's SQL query and returns the complete results. Unlike `preview_insight` (limited to ~20 activities), this returns ALL data matching your query.
66
+
67
+ **Think of it as:**
68
+ - Running a saved SQL query
69
+ - Executing a database view
70
+ - Generating a report
71
+ - Exporting query results
72
+
73
+ **When to Use:**
74
+ - View full insight results
75
+ - Export data for analysis
76
+ - Verify insight after creation
77
+ - Check updated data after workflow changes
78
+ - Generate reports
79
+ - Feed data to other tools
80
+
81
+ **When NOT to Use:**
82
+ - Testing queries (use preview_insight)
83
+ - Need just a few records (use list_activities)
84
+ - Real-time data not needed (results may be cached)
85
+ - Insight doesn't exist yet (create it first)
86
+
87
+ ## Core Concepts
88
+
89
+ ### Insight Execution Flow
90
+
91
+ ```
92
+ 1. Insight Created
93
+
94
+ 2. get_insight_data() called
95
+
96
+ 3. Check cache
97
+
98
+ 4a. Cached? → Return cached results (fast)
99
+ 4b. Not cached/update=true? → Execute SQL (slower)
100
+
101
+ 5. Return columns + rows
102
+ ```
103
+
104
+ ### Data vs Preview
105
+
106
+ | Feature | preview_insight | get_insight_data |
107
+ |---------|----------------|------------------|
108
+ | Requires saved insight | ❌ No | ✅ Yes |
109
+ | Data limit | ~20 activities | All activities |
110
+ | Speed | Very fast | Depends on data size |
111
+ | Use case | Testing | Production reports |
112
+ | Caching | Not cached | Can use cache |
113
+
114
+ ### Cached vs Fresh Data
115
+
116
+ **Cached Results (update: false, default):**
117
+ - Fast retrieval
118
+ - May be stale if activities changed
119
+ - Good for static reports
120
+ - Recommended for frequent access
121
+
122
+ **Fresh Results (update: true):**
123
+ - Recalculates from current data
124
+ - Slower execution
125
+ - Always current
126
+ - Use after workflow updates
127
+
128
+ ## Basic Usage
129
+
130
+ ### Example 1: Get Cached Results
131
+
132
+ ```javascript
133
+ // Get results from saved insight (uses cache)
134
+ get_insight_data({
135
+ insightId: '68446dc05b30685f67c6fcd4'
136
+ })
137
+
138
+ // Response:
139
+ // ✅ Insight Data Retrieved
140
+ // Insight ID: 68446dc05b30685f67c6fcd4
141
+ // Updated: No (cached)
142
+ //
143
+ // Results:
144
+ // - Rows: 1523
145
+ // - Columns: 3
146
+ //
147
+ // Data Preview (first 50 rows):
148
+ // | title | priority | dueDate |
149
+ // |-------|----------|---------|
150
+ // | Task 1 | High | 2024-12-31 |
151
+ // | Task 2 | Medium | 2024-12-25 |
152
+ // ...
153
+ ```
154
+
155
+ ### Example 2: Force Refresh
156
+
157
+ ```javascript
158
+ // Recalculate with current activity data
159
+ get_insight_data({
160
+ insightId: '68446dc05b30685f67c6fcd4',
161
+ update: true // Recalculate
162
+ })
163
+
164
+ // Response:
165
+ // ✅ Insight Data Retrieved
166
+ // Insight ID: 68446dc05b30685f67c6fcd4
167
+ // Updated: Yes (recalculated)
168
+ //
169
+ // Results:
170
+ // - Rows: 1547 // New activities added since cache
171
+ // - Columns: 3
172
+ ```
173
+
174
+ ### Example 3: Complete Workflow
175
+
176
+ ```javascript
177
+ // 1. Create insight
178
+ const result = create_insight({
179
+ name: 'High Priority Tasks',
180
+ sources: [{
181
+ name: 'tasks',
182
+ workflowId: 'tasks-workflow-id',
183
+ fields: [
184
+ { name: 'title', meta: 'name' },
185
+ { name: 'priority', fieldId: 'priority-field-id' }
186
+ ]
187
+ }],
188
+ query: 'SELECT title, priority FROM tasks WHERE priority = "High"'
189
+ })
190
+
191
+ // 2. Get insight data
192
+ const data = get_insight_data({
193
+ insightId: result.insightId
194
+ })
195
+
196
+ // 3. Process results
197
+ console.log(`Found ${data.rows.length} high priority tasks`)
198
+ data.rows.forEach(row => {
199
+ const [title, priority] = row
200
+ console.log(`- ${title} (${priority})`)
201
+ })
202
+ ```
203
+
204
+ ## Update vs Cached Data
205
+
206
+ ### When to Use Cached Results (update: false)
207
+
208
+ **Scenarios:**
209
+ - Reading same insight multiple times
210
+ - Data doesn't change frequently
211
+ - Performance is critical
212
+ - Report is for historical reference
213
+ - Running scheduled reports
214
+
215
+ **Example:**
216
+ ```javascript
217
+ // Dashboard that refreshes every minute
218
+ setInterval(() => {
219
+ const data = get_insight_data({
220
+ insightId: 'dashboard-insight-id'
221
+ // No update - uses cache (fast)
222
+ })
223
+ updateDashboard(data)
224
+ }, 60000)
225
+ ```
226
+
227
+ ### When to Force Update (update: true)
228
+
229
+ **Scenarios:**
230
+ - After creating new activities
231
+ - After updating activity fields
232
+ - Need real-time accuracy
233
+ - First time executing insight
234
+ - Data known to have changed
235
+ - Debugging data issues
236
+
237
+ **Example:**
238
+ ```javascript
239
+ // 1. Create activities
240
+ create_activity({
241
+ workflowId: 'tasks-workflow-id',
242
+ name: 'New Task',
243
+ fields: { priority: 'High' }
244
+ })
245
+
246
+ // 2. Get fresh data (includes new activity)
247
+ get_insight_data({
248
+ insightId: 'tasks-insight-id',
249
+ update: true // Force recalculation
250
+ })
251
+ ```
252
+
253
+ ### Hybrid Approach
254
+
255
+ ```javascript
256
+ // Check last update time, refresh if old
257
+ function getInsightDataSmart(insightId) {
258
+ const insights = list_insights()
259
+ const insight = insights.find(i => i._id === insightId)
260
+
261
+ const lastUpdate = new Date(insight.lastCalculated)
262
+ const ageMinutes = (Date.now() - lastUpdate) / 1000 / 60
263
+
264
+ // Refresh if older than 5 minutes
265
+ const shouldUpdate = ageMinutes > 5
266
+
267
+ return get_insight_data({
268
+ insightId: insightId,
269
+ update: shouldUpdate
270
+ })
271
+ }
272
+ ```
273
+
274
+ ## Working with Results
275
+
276
+ ### Result Structure
277
+
278
+ ```javascript
279
+ const result = get_insight_data({
280
+ insightId: 'insight-id'
281
+ })
282
+
283
+ // Structure:
284
+ // {
285
+ // columns: ['column1', 'column2', 'column3'],
286
+ // rows: [
287
+ // ['value1a', 'value2a', 'value3a'],
288
+ // ['value1b', 'value2b', 'value3b'],
289
+ // ...
290
+ // ]
291
+ // }
292
+ ```
293
+
294
+ ### Processing Results
295
+
296
+ **Example 1: Convert to Objects**
297
+
298
+ ```javascript
299
+ const data = get_insight_data({
300
+ insightId: 'insight-id'
301
+ })
302
+
303
+ // Convert to array of objects
304
+ const objects = data.rows.map(row => {
305
+ const obj = {}
306
+ data.columns.forEach((col, i) => {
307
+ obj[col] = row[i]
308
+ })
309
+ return obj
310
+ })
311
+
312
+ console.log(objects)
313
+ // [
314
+ // { title: 'Task 1', priority: 'High', dueDate: '2024-12-31' },
315
+ // { title: 'Task 2', priority: 'Medium', dueDate: '2024-12-25' },
316
+ // ...
317
+ // ]
318
+ ```
319
+
320
+ **Example 2: Filter Results**
321
+
322
+ ```javascript
323
+ const data = get_insight_data({
324
+ insightId: 'insight-id'
325
+ })
326
+
327
+ // Find rows where column 2 (priority) is 'High'
328
+ const highPriorityRows = data.rows.filter(row => row[1] === 'High')
329
+
330
+ console.log(`Found ${highPriorityRows.length} high priority items`)
331
+ ```
332
+
333
+ **Example 3: Group by Column**
334
+
335
+ ```javascript
336
+ const data = get_insight_data({
337
+ insightId: 'insight-id'
338
+ })
339
+
340
+ // Group by priority (column index 1)
341
+ const grouped = {}
342
+ data.rows.forEach(row => {
343
+ const priority = row[1]
344
+ if (!grouped[priority]) {
345
+ grouped[priority] = []
346
+ }
347
+ grouped[priority].push(row)
348
+ })
349
+
350
+ console.log('Tasks by priority:', grouped)
351
+ // {
352
+ // 'High': [[...], [...]],
353
+ // 'Medium': [[...], [...], ...],
354
+ // 'Low': [[...]]
355
+ // }
356
+ ```
357
+
358
+ **Example 4: Export to CSV**
359
+
360
+ ```javascript
361
+ const data = get_insight_data({
362
+ insightId: 'insight-id'
363
+ })
364
+
365
+ // Generate CSV
366
+ let csv = data.columns.join(',') + '\n'
367
+ data.rows.forEach(row => {
368
+ csv += row.map(cell => `"${cell}"`).join(',') + '\n'
369
+ })
370
+
371
+ console.log(csv)
372
+ // title,priority,dueDate
373
+ // "Task 1","High","2024-12-31"
374
+ // "Task 2","Medium","2024-12-25"
375
+ // ...
376
+ ```
377
+
378
+ **Example 5: Calculate Statistics**
379
+
380
+ ```javascript
381
+ const data = get_insight_data({
382
+ insightId: 'budget-insight-id'
383
+ })
384
+
385
+ // Assuming column 0 is 'projectName', column 1 is 'budget'
386
+ const budgets = data.rows.map(row => parseFloat(row[1]))
387
+
388
+ const total = budgets.reduce((sum, b) => sum + b, 0)
389
+ const avg = total / budgets.length
390
+ const max = Math.max(...budgets)
391
+ const min = Math.min(...budgets)
392
+
393
+ console.log(`Total Budget: $${total}`)
394
+ console.log(`Average Budget: $${avg}`)
395
+ console.log(`Max Budget: $${max}`)
396
+ console.log(`Min Budget: $${min}`)
397
+ ```
398
+
399
+ ## Large Datasets
400
+
401
+ ### Handling Many Rows
402
+
403
+ **Insight returns thousands of rows:**
404
+
405
+ ```javascript
406
+ const data = get_insight_data({
407
+ insightId: 'large-insight-id'
408
+ })
409
+
410
+ console.log(`Retrieved ${data.rows.length} rows`)
411
+ // Retrieved 15,247 rows
412
+
413
+ // Process in batches
414
+ const batchSize = 100
415
+ for (let i = 0; i < data.rows.length; i += batchSize) {
416
+ const batch = data.rows.slice(i, i + batchSize)
417
+ processBatch(batch)
418
+ }
419
+ ```
420
+
421
+ ### Pagination Strategy
422
+
423
+ **Use SQL LIMIT/OFFSET in insight query:**
424
+
425
+ ```javascript
426
+ // Create paginated insight
427
+ create_insight({
428
+ name: 'Tasks Page 1',
429
+ sources: [...],
430
+ query: 'SELECT * FROM tasks ORDER BY created DESC LIMIT 100 OFFSET 0'
431
+ })
432
+
433
+ create_insight({
434
+ name: 'Tasks Page 2',
435
+ sources: [...],
436
+ query: 'SELECT * FROM tasks ORDER BY created DESC LIMIT 100 OFFSET 100'
437
+ })
438
+
439
+ // Get pages
440
+ const page1 = get_insight_data({ insightId: 'page1-id' })
441
+ const page2 = get_insight_data({ insightId: 'page2-id' })
442
+ ```
443
+
444
+ ### Memory Considerations
445
+
446
+ ```javascript
447
+ // For very large datasets, process incrementally
448
+ const data = get_insight_data({
449
+ insightId: 'huge-insight-id'
450
+ })
451
+
452
+ // Don't store all processed data in memory
453
+ let processed = 0
454
+ data.rows.forEach(row => {
455
+ // Process row immediately
456
+ processRow(row)
457
+ processed++
458
+
459
+ // Don't accumulate results
460
+ })
461
+
462
+ console.log(`Processed ${processed} rows`)
463
+ ```
464
+
465
+ ## Common Scenarios
466
+
467
+ ### Scenario 1: Daily Report Generation
468
+
469
+ ```javascript
470
+ // Generate daily high priority tasks report
471
+
472
+ // 1. Get insight data (fresh)
473
+ const data = get_insight_data({
474
+ insightId: 'daily-tasks-insight-id',
475
+ update: true // Fresh data
476
+ })
477
+
478
+ // 2. Format as report
479
+ console.log('=== Daily High Priority Tasks Report ===')
480
+ console.log(`Generated: ${new Date().toISOString()}`)
481
+ console.log(`Total Tasks: ${data.rows.length}\n`)
482
+
483
+ // 3. Output table
484
+ console.log(data.columns.join(' | '))
485
+ console.log(data.columns.map(() => '---').join(' | '))
486
+ data.rows.forEach(row => {
487
+ console.log(row.join(' | '))
488
+ })
489
+ ```
490
+
491
+ ### Scenario 2: Data Export
492
+
493
+ ```javascript
494
+ // Export insight data to JSON file
495
+
496
+ const data = get_insight_data({
497
+ insightId: 'export-insight-id',
498
+ update: true
499
+ })
500
+
501
+ // Convert to objects
502
+ const records = data.rows.map(row => {
503
+ const record = {}
504
+ data.columns.forEach((col, i) => {
505
+ record[col] = row[i]
506
+ })
507
+ return record
508
+ })
509
+
510
+ // Save to file (pseudo-code)
511
+ const json = JSON.stringify(records, null, 2)
512
+ saveToFile('export.json', json)
513
+
514
+ console.log(`Exported ${records.length} records`)
515
+ ```
516
+
517
+ ### Scenario 3: Verify Insight After Creation
518
+
519
+ ```javascript
520
+ // 1. Create insight
521
+ const result = create_insight({
522
+ name: 'New Insight',
523
+ sources: [...],
524
+ query: 'SELECT ...'
525
+ })
526
+
527
+ // 2. Immediately get data to verify
528
+ const data = get_insight_data({
529
+ insightId: result.insightId,
530
+ update: true // Force calculation
531
+ })
532
+
533
+ // 3. Verify results
534
+ if (data.rows.length === 0) {
535
+ console.warn('⚠️ Insight returns no data!')
536
+ } else {
537
+ console.log(`✅ Insight working: ${data.rows.length} rows`)
538
+ }
539
+ ```
540
+
541
+ ### Scenario 4: Compare Before/After
542
+
543
+ ```javascript
544
+ // Check data before and after workflow changes
545
+
546
+ // Before changes
547
+ const before = get_insight_data({
548
+ insightId: 'insight-id',
549
+ update: true
550
+ })
551
+ console.log('Before:', before.rows.length, 'rows')
552
+
553
+ // Make changes
554
+ create_activity({ ... })
555
+ update_activity({ ... })
556
+
557
+ // After changes (force refresh)
558
+ const after = get_insight_data({
559
+ insightId: 'insight-id',
560
+ update: true // Fresh data
561
+ })
562
+ console.log('After:', after.rows.length, 'rows')
563
+ console.log('Difference:', after.rows.length - before.rows.length)
564
+ ```
565
+
566
+ ### Scenario 5: Dashboard Data
567
+
568
+ ```javascript
569
+ // Get multiple insight datasets for dashboard
570
+
571
+ async function getDashboardData() {
572
+ // Get all insights (can run in parallel)
573
+ const [tasksData, projectsData, statsData] = await Promise.all([
574
+ get_insight_data({ insightId: 'tasks-insight-id' }),
575
+ get_insight_data({ insightId: 'projects-insight-id' }),
576
+ get_insight_data({ insightId: 'stats-insight-id' })
577
+ ])
578
+
579
+ return {
580
+ tasks: {
581
+ total: tasksData.rows.length,
582
+ data: tasksData.rows
583
+ },
584
+ projects: {
585
+ total: projectsData.rows.length,
586
+ data: projectsData.rows
587
+ },
588
+ stats: {
589
+ data: statsData.rows
590
+ }
591
+ }
592
+ }
593
+ ```
594
+
595
+ ### Scenario 6: Find Specific Records
596
+
597
+ ```javascript
598
+ // Use insight to find specific records
599
+
600
+ const data = get_insight_data({
601
+ insightId: 'all-tasks-insight-id'
602
+ })
603
+
604
+ // Find specific task by title (column 0)
605
+ const task = data.rows.find(row => row[0] === 'Important Task')
606
+
607
+ if (task) {
608
+ console.log('Found task:', task)
609
+ } else {
610
+ console.log('Task not found')
611
+ }
612
+ ```
613
+
614
+ ## Performance Considerations
615
+
616
+ ### Execution Time
617
+
618
+ **Fast insights (< 1 second):**
619
+ - Simple SELECT with WHERE
620
+ - Single workflow source
621
+ - Small result set (< 1000 rows)
622
+ - Using cached results
623
+
624
+ **Slow insights (> 5 seconds):**
625
+ - Complex JOINs (3+ workflows)
626
+ - Large result sets (> 10,000 rows)
627
+ - Complex aggregations
628
+ - Force update on large dataset
629
+
630
+ ### Optimization Tips
631
+
632
+ **1. Use cached results when possible:**
633
+ ```javascript
634
+ // Fast - uses cache
635
+ get_insight_data({ insightId: 'id' })
636
+
637
+ // Slow - recalculates
638
+ get_insight_data({ insightId: 'id', update: true })
639
+ ```
640
+
641
+ **2. Limit result sets in SQL:**
642
+ ```javascript
643
+ // Better: Limited results
644
+ create_insight({
645
+ query: 'SELECT * FROM tasks LIMIT 1000'
646
+ })
647
+
648
+ // Worse: All results
649
+ create_insight({
650
+ query: 'SELECT * FROM tasks' // Could be 50,000 rows
651
+ })
652
+ ```
653
+
654
+ **3. Use aggregations in SQL:**
655
+ ```javascript
656
+ // Better: Aggregate in SQL
657
+ create_insight({
658
+ query: `
659
+ SELECT priority, COUNT(*) as count
660
+ FROM tasks
661
+ GROUP BY priority
662
+ `
663
+ })
664
+ const data = get_insight_data({ insightId: 'id' })
665
+ // Returns: 3 rows (one per priority)
666
+
667
+ // Worse: Aggregate in code
668
+ create_insight({
669
+ query: 'SELECT priority FROM tasks'
670
+ })
671
+ const data = get_insight_data({ insightId: 'id' })
672
+ // Returns: 10,000 rows, then count in code
673
+ ```
674
+
675
+ **4. Schedule expensive updates:**
676
+ ```javascript
677
+ // Update heavy insights during off-hours
678
+ if (new Date().getHours() < 6) { // 12am - 6am
679
+ get_insight_data({
680
+ insightId: 'expensive-insight-id',
681
+ update: true
682
+ })
683
+ }
684
+ ```
685
+
686
+ ## Best Practices
687
+
688
+ ### 1. Verify Insight Exists
689
+
690
+ ```javascript
691
+ // Check insight exists before getting data
692
+ const insights = list_insights()
693
+ const insight = insights.find(i => i._id === 'target-id')
694
+
695
+ if (!insight) {
696
+ console.error('Insight not found')
697
+ return
698
+ }
699
+
700
+ const data = get_insight_data({
701
+ insightId: insight._id
702
+ })
703
+ ```
704
+
705
+ ### 2. Handle Empty Results
706
+
707
+ ```javascript
708
+ const data = get_insight_data({
709
+ insightId: 'insight-id'
710
+ })
711
+
712
+ if (!data.rows || data.rows.length === 0) {
713
+ console.warn('No data returned')
714
+ return
715
+ }
716
+
717
+ // Process data
718
+ console.log(`Processing ${data.rows.length} rows`)
719
+ ```
720
+
721
+ ### 3. Use update: true After Data Changes
722
+
723
+ ```javascript
724
+ // Workflow: Create activities → Get fresh data
725
+
726
+ // 1. Create activities
727
+ create_activity({ ... })
728
+ create_activity({ ... })
729
+
730
+ // 2. Get fresh insight data
731
+ const data = get_insight_data({
732
+ insightId: 'insight-id',
733
+ update: true // Include new activities
734
+ })
735
+ ```
736
+
737
+ ### 4. Cache Results Locally When Appropriate
738
+
739
+ ```javascript
740
+ // Cache insight data for repeated access
741
+ let cachedData = null
742
+ let cacheTime = null
743
+
744
+ function getInsightDataCached(insightId) {
745
+ const now = Date.now()
746
+ const cacheMaxAge = 5 * 60 * 1000 // 5 minutes
747
+
748
+ if (cachedData && cacheTime && (now - cacheTime < cacheMaxAge)) {
749
+ console.log('Using local cache')
750
+ return cachedData
751
+ }
752
+
753
+ console.log('Fetching fresh data')
754
+ cachedData = get_insight_data({ insightId })
755
+ cacheTime = now
756
+
757
+ return cachedData
758
+ }
759
+ ```
760
+
761
+ ### 5. Validate Column Structure
762
+
763
+ ```javascript
764
+ const data = get_insight_data({
765
+ insightId: 'insight-id'
766
+ })
767
+
768
+ // Verify expected columns
769
+ const expectedColumns = ['title', 'priority', 'dueDate']
770
+ const hasAllColumns = expectedColumns.every(col =>
771
+ data.columns.includes(col)
772
+ )
773
+
774
+ if (!hasAllColumns) {
775
+ console.error('Unexpected column structure:', data.columns)
776
+ return
777
+ }
778
+
779
+ // Safe to process
780
+ processData(data)
781
+ ```
782
+
783
+ ### 6. Handle NULL Values
784
+
785
+ ```javascript
786
+ const data = get_insight_data({
787
+ insightId: 'insight-id'
788
+ })
789
+
790
+ // Convert rows, handle NULL
791
+ const records = data.rows.map(row => {
792
+ return data.columns.reduce((obj, col, i) => {
793
+ obj[col] = row[i] ?? 'N/A' // Replace NULL with 'N/A'
794
+ return obj
795
+ }, {})
796
+ })
797
+ ```
798
+
799
+ ### 7. Log Data Retrieval
800
+
801
+ ```javascript
802
+ console.log(`Fetching insight data: ${insightId}`)
803
+ console.log(`Update mode: ${update ? 'Fresh' : 'Cached'}`)
804
+
805
+ const startTime = Date.now()
806
+ const data = get_insight_data({
807
+ insightId: insightId,
808
+ update: update
809
+ })
810
+ const elapsed = Date.now() - startTime
811
+
812
+ console.log(`Retrieved ${data.rows.length} rows in ${elapsed}ms`)
813
+ console.log(`Columns: ${data.columns.join(', ')}`)
814
+ ```
815
+
816
+ ## Troubleshooting
817
+
818
+ ### Error: "Insight Not Found"
819
+
820
+ **Cause:** Invalid insight ID or insight deleted.
821
+
822
+ **Solution:**
823
+
824
+ ```javascript
825
+ // 1. List insights to find correct ID
826
+ const insights = list_insights()
827
+ console.log('Available insights:')
828
+ insights.forEach(i => {
829
+ console.log(`- ${i.name}: ${i._id}`)
830
+ })
831
+
832
+ // 2. Use correct ID
833
+ get_insight_data({
834
+ insightId: 'correct-insight-id'
835
+ })
836
+ ```
837
+
838
+ ### Error: "Permission Denied"
839
+
840
+ **Cause:** Don't have access to insight or underlying workflows.
841
+
842
+ **Solution:**
843
+ - Check insight permissions
844
+ - Verify access to workflows in insight sources
845
+ - Contact insight owner or workspace admin
846
+
847
+ ### Returns Empty Data
848
+
849
+ **Causes:**
850
+ 1. No activities match query
851
+ 2. Insight SQL has WHERE clause too restrictive
852
+ 3. JOINs eliminate all rows
853
+
854
+ **Solutions:**
855
+
856
+ ```javascript
857
+ // 1. Check if cached results are stale
858
+ get_insight_data({
859
+ insightId: 'insight-id',
860
+ update: true // Force fresh calculation
861
+ })
862
+
863
+ // 2. Verify activities exist
864
+ list_activities({
865
+ workflowId: 'workflow-id',
866
+ limit: 1
867
+ })
868
+
869
+ // 3. Test with preview_insight
870
+ preview_insight({
871
+ sources: [...],
872
+ query: '...'
873
+ })
874
+ ```
875
+
876
+ ### Data Seems Outdated
877
+
878
+ **Cause:** Using cached results, activities changed.
879
+
880
+ **Solution:**
881
+
882
+ ```javascript
883
+ // Force recalculation
884
+ get_insight_data({
885
+ insightId: 'insight-id',
886
+ update: true // Get current data
887
+ })
888
+ ```
889
+
890
+ ### Very Slow Execution
891
+
892
+ **Causes:**
893
+ 1. Large dataset
894
+ 2. Complex JOINs
895
+ 3. Server load
896
+
897
+ **Solutions:**
898
+
899
+ ```javascript
900
+ // 1. Use cached results
901
+ get_insight_data({
902
+ insightId: 'insight-id'
903
+ // Don't use update: true
904
+ })
905
+
906
+ // 2. Limit results in SQL
907
+ // Modify insight query to include LIMIT
908
+
909
+ // 3. Use aggregation instead of detail
910
+ // Change insight to GROUP BY instead of returning all rows
911
+ ```
912
+
913
+ ### Unexpected Column Structure
914
+
915
+ **Cause:** Insight query changed or different than expected.
916
+
917
+ **Solution:**
918
+
919
+ ```javascript
920
+ const data = get_insight_data({
921
+ insightId: 'insight-id'
922
+ })
923
+
924
+ // Check actual columns
925
+ console.log('Actual columns:', data.columns)
926
+
927
+ // Adapt processing
928
+ data.rows.forEach(row => {
929
+ const record = {}
930
+ data.columns.forEach((col, i) => {
931
+ record[col] = row[i]
932
+ })
933
+ console.log(record)
934
+ })
935
+ ```
936
+
937
+ ## Integration with Other Tools
938
+
939
+ ### With create_insight
940
+
941
+ Standard workflow:
942
+
943
+ ```javascript
944
+ // 1. Create insight
945
+ const result = create_insight({
946
+ name: 'My Report',
947
+ sources: [...],
948
+ query: '...'
949
+ })
950
+
951
+ // 2. Get data
952
+ const data = get_insight_data({
953
+ insightId: result.insightId,
954
+ update: true // Force initial calculation
955
+ })
956
+ ```
957
+
958
+ ### With preview_insight
959
+
960
+ Test then execute:
961
+
962
+ ```javascript
963
+ // 1. Preview query
964
+ preview_insight({
965
+ sources: [...],
966
+ query: '...'
967
+ })
968
+ // ✅ Works!
969
+
970
+ // 2. Create insight
971
+ const result = create_insight({
972
+ name: 'Report',
973
+ sources: [...],
974
+ query: '...'
975
+ })
976
+
977
+ // 3. Get full data
978
+ const data = get_insight_data({
979
+ insightId: result.insightId
980
+ })
981
+ ```
982
+
983
+ ### With list_insights
984
+
985
+ Find and execute:
986
+
987
+ ```javascript
988
+ // 1. List insights
989
+ const insights = list_insights()
990
+
991
+ // 2. Find specific insight
992
+ const insight = insights.find(i => i.name === 'Daily Report')
993
+
994
+ // 3. Get data
995
+ const data = get_insight_data({
996
+ insightId: insight._id,
997
+ update: true
998
+ })
999
+ ```
1000
+
1001
+ ### With list_activities
1002
+
1003
+ Compare results:
1004
+
1005
+ ```javascript
1006
+ // Get insight data
1007
+ const insightData = get_insight_data({
1008
+ insightId: 'tasks-high-priority-id'
1009
+ })
1010
+
1011
+ // Compare with list_activities
1012
+ const activities = list_activities({
1013
+ workflowId: 'tasks-workflow-id',
1014
+ filters: [{ field: 'priority-field-id', operator: 'eq', value: 'High' }]
1015
+ })
1016
+
1017
+ console.log('Insight rows:', insightData.rows.length)
1018
+ console.log('Activities:', activities.items.length)
1019
+ // Should match
1020
+ ```
1021
+
1022
+ ## Summary
1023
+
1024
+ **Key Takeaways:**
1025
+ 1. 📊 **Full results** - Returns ALL matching activities (not limited)
1026
+ 2. ⚡ **Cached by default** - Fast with cached results
1027
+ 3. 🔄 **Force update** - Use update: true for fresh data
1028
+ 4. 📋 **Tabular format** - Returns columns + rows arrays
1029
+ 5. 🎯 **Production use** - For reports and data exports
1030
+ 6. 📈 **Large datasets** - Can return thousands of rows
1031
+ 7. ✅ **After preview** - Create insight first, then get data
1032
+
1033
+ **Quick Decision Tree:**
1034
+
1035
+ ```
1036
+ Need insight data?
1037
+
1038
+ ├─ Testing query? → Use preview_insight
1039
+ ├─ Insight not created? → Create first with create_insight
1040
+ ├─ Need current data? → Use update: true
1041
+ ├─ Performance critical? → Use cached (update: false)
1042
+ ├─ Large dataset? → Consider SQL LIMIT or aggregation
1043
+ └─ For export? → Convert rows to desired format
1044
+ ```
1045
+
1046
+ ## Additional Resources
1047
+
1048
+ - See `create-insight-skill` for creating insights
1049
+ - See `preview-insight-skill` for testing queries
1050
+ - See `remove-insight-skill` for deleting insights
1051
+ - See `insight-api` skill for comprehensive Insight API documentation
1052
+ - Use `list_insights` to find insight IDs
1053
+ - Use `list_activities` to compare results