node-red-contrib-knx-ultimate 6.3.15 → 6.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/nodes/knxUltimateAI.html +99 -0
- package/nodes/knxUltimateAI.js +736 -27
- package/nodes/locales/de/knxUltimateAI.html +5 -0
- package/nodes/locales/de/knxUltimateAI.json +7 -0
- package/nodes/locales/en/knxUltimateAI.html +5 -0
- package/nodes/locales/en/knxUltimateAI.json +7 -0
- package/nodes/locales/es/knxUltimateAI.html +5 -0
- package/nodes/locales/es/knxUltimateAI.json +7 -0
- package/nodes/locales/fr/knxUltimateAI.html +5 -0
- package/nodes/locales/fr/knxUltimateAI.json +7 -0
- package/nodes/locales/it/knxUltimateAI.html +5 -0
- package/nodes/locales/it/knxUltimateAI.json +7 -0
- package/nodes/locales/zh-CN/knxUltimateAI.html +5 -0
- package/nodes/locales/zh-CN/knxUltimateAI.json +7 -0
- package/nodes/utils/knxAiCamera.js +302 -0
- package/nodes/utils/knxAiChatContext.js +85 -5
- package/package.json +1 -1
- package/resources/KNXAIChatAdapterMappings.js +64 -1
package/nodes/knxUltimateAI.js
CHANGED
|
@@ -24,15 +24,30 @@ const {
|
|
|
24
24
|
} = require('./utils/knxAiHomeMemory')
|
|
25
25
|
const {
|
|
26
26
|
CHAT_CONTEXT_MAX_BYTES,
|
|
27
|
+
addKnxAiCameraWatch,
|
|
27
28
|
addKnxAiChatTurn,
|
|
28
29
|
buildKnxAiChatContextMarkdown,
|
|
29
30
|
buildKnxAiChatPromptContext,
|
|
30
31
|
clearKnxAiChatSession,
|
|
31
32
|
conversationMapFromKnxAiChatContext,
|
|
32
33
|
createEmptyKnxAiChatContext,
|
|
34
|
+
listAllKnxAiCameraWatches,
|
|
35
|
+
listKnxAiCameraWatches,
|
|
33
36
|
normalizeKnxAiChatContext,
|
|
34
|
-
parseKnxAiChatContextMarkdown
|
|
37
|
+
parseKnxAiChatContextMarkdown,
|
|
38
|
+
removeKnxAiCameraWatches
|
|
35
39
|
} = require('./utils/knxAiChatContext')
|
|
40
|
+
const {
|
|
41
|
+
buildKnxAiCameraNotificationText,
|
|
42
|
+
cameraWatchMatchesEvent,
|
|
43
|
+
getKnxAiCameraAdapterRegistry,
|
|
44
|
+
normalizeKnxAiCameraActions,
|
|
45
|
+
normalizeKnxAiCameraEvent,
|
|
46
|
+
normalizeKnxAiCameraImage,
|
|
47
|
+
normalizeKnxAiCameraRegistration,
|
|
48
|
+
normalizeSearchText,
|
|
49
|
+
resolveKnxAiCamera
|
|
50
|
+
} = require('./utils/knxAiCamera')
|
|
36
51
|
let googleTranslateTTS = null
|
|
37
52
|
try {
|
|
38
53
|
googleTranslateTTS = require('google-translate-tts')
|
|
@@ -48,6 +63,45 @@ const sharedKnxAiHomeMemoryStores = new Map()
|
|
|
48
63
|
const sharedKnxAiChatContextStores = new Map()
|
|
49
64
|
const knxAiVueDistDir = path.join(__dirname, 'plugins', 'knxUltimateAI-vue')
|
|
50
65
|
|
|
66
|
+
const summarizeDetectedKnxAiCameraAdapters = ({ registry, node } = {}) => {
|
|
67
|
+
const sourceRegistry = registry || getKnxAiCameraAdapterRegistry()
|
|
68
|
+
const adapters = new Map(sourceRegistry && sourceRegistry.adapters instanceof Map ? sourceRegistry.adapters : [])
|
|
69
|
+
const providers = new Map(sourceRegistry && sourceRegistry.providers instanceof Map ? sourceRegistry.providers : [])
|
|
70
|
+
const cameras = new Map()
|
|
71
|
+
|
|
72
|
+
if (node && node._cameraAdapters instanceof Map) {
|
|
73
|
+
node._cameraAdapters.forEach((adapter, id) => adapters.set(String(id), adapter))
|
|
74
|
+
}
|
|
75
|
+
if (node && node._cameraProviders instanceof Map) {
|
|
76
|
+
node._cameraProviders.forEach((provider, id) => providers.set(String(id), provider))
|
|
77
|
+
}
|
|
78
|
+
if (node && node._cameraCatalog instanceof Map) {
|
|
79
|
+
node._cameraCatalog.forEach((camera, id) => cameras.set(String(id), camera))
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return Array.from(adapters.entries()).map(([adapterId, value]) => {
|
|
83
|
+
const adapter = value && typeof value === 'object' ? value : {}
|
|
84
|
+
const matchingProviderIds = new Set()
|
|
85
|
+
providers.forEach((provider, providerId) => {
|
|
86
|
+
if (String(provider && provider.adapterId || '') === adapterId) matchingProviderIds.add(String(providerId))
|
|
87
|
+
})
|
|
88
|
+
let cameraCount = 0
|
|
89
|
+
cameras.forEach(camera => {
|
|
90
|
+
const cameraAdapterId = String(camera && camera.adapterId || '')
|
|
91
|
+
const cameraProviderId = String(camera && camera.providerId || '')
|
|
92
|
+
if (cameraAdapterId === adapterId || matchingProviderIds.has(cameraProviderId)) cameraCount += 1
|
|
93
|
+
})
|
|
94
|
+
return {
|
|
95
|
+
id: adapterId,
|
|
96
|
+
title: String(adapter.title || adapterId),
|
|
97
|
+
packageName: String(adapter.packageName || ''),
|
|
98
|
+
capabilities: Array.isArray(adapter.capabilities) ? adapter.capabilities.map(String) : [],
|
|
99
|
+
providerCount: matchingProviderIds.size,
|
|
100
|
+
cameraCount
|
|
101
|
+
}
|
|
102
|
+
}).sort((left, right) => left.title.localeCompare(right.title))
|
|
103
|
+
}
|
|
104
|
+
|
|
51
105
|
const bindSharedKnxAiState = ({ registry, filePath, node, property, initialValue }) => {
|
|
52
106
|
let store = registry.get(filePath)
|
|
53
107
|
if (!store) {
|
|
@@ -819,7 +873,12 @@ const parseKnxAiConversationResponse = (value) => {
|
|
|
819
873
|
: parsed.locale !== undefined
|
|
820
874
|
? parsed.locale
|
|
821
875
|
: '').trim()
|
|
822
|
-
|
|
876
|
+
const cameraActions = Array.isArray(parsed.cameraActions)
|
|
877
|
+
? parsed.cameraActions
|
|
878
|
+
: Array.isArray(parsed.camera_actions)
|
|
879
|
+
? parsed.camera_actions
|
|
880
|
+
: []
|
|
881
|
+
return { reply, commands, cameraActions, language }
|
|
823
882
|
}
|
|
824
883
|
|
|
825
884
|
const extractKnxAiQuestion = (msg) => {
|
|
@@ -1182,6 +1241,36 @@ const executeKnxAiChatAdapter = ({
|
|
|
1182
1241
|
return result
|
|
1183
1242
|
}
|
|
1184
1243
|
|
|
1244
|
+
const applyKnxAiChatMediaPresetFallback = ({ preset, message, inputMessage } = {}) => {
|
|
1245
|
+
if (String(preset || '') !== 'windkh-telegrambot' || !message || typeof message !== 'object') return message
|
|
1246
|
+
const image = message.knxAi && message.knxAi.image
|
|
1247
|
+
if (!image || !Buffer.isBuffer(image.data)) return message
|
|
1248
|
+
if (message.payload && typeof message.payload === 'object' && message.payload.type === 'photo') return message
|
|
1249
|
+
|
|
1250
|
+
const source = inputMessage && typeof inputMessage === 'object'
|
|
1251
|
+
? inputMessage
|
|
1252
|
+
: message.inputMessage && typeof message.inputMessage === 'object'
|
|
1253
|
+
? message.inputMessage
|
|
1254
|
+
: message
|
|
1255
|
+
const sourcePayload = source.payload && typeof source.payload === 'object' ? source.payload : {}
|
|
1256
|
+
const chatId = sourcePayload.chatId !== undefined ? sourcePayload.chatId : source.chatId
|
|
1257
|
+
if (chatId === undefined || chatId === null || chatId === '') return message
|
|
1258
|
+
|
|
1259
|
+
let caption = message.payload
|
|
1260
|
+
if (caption && typeof caption === 'object') caption = caption.error || caption.message || ''
|
|
1261
|
+
caption = String(caption === undefined || caption === null ? '' : caption).slice(0, 1024)
|
|
1262
|
+
const filename = String(image.filename || 'camera-snapshot.jpg').trim().slice(0, 240) || 'camera-snapshot.jpg'
|
|
1263
|
+
const contentType = String(image.mediaType || 'image/jpeg').split(';')[0].trim().toLowerCase() || 'image/jpeg'
|
|
1264
|
+
message.payload = {
|
|
1265
|
+
chatId,
|
|
1266
|
+
type: 'photo',
|
|
1267
|
+
content: image.data,
|
|
1268
|
+
options: caption ? { caption } : {},
|
|
1269
|
+
fileOptions: { filename, contentType }
|
|
1270
|
+
}
|
|
1271
|
+
return message
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1185
1274
|
const buildKnxAiUniversalMessage = ({
|
|
1186
1275
|
command,
|
|
1187
1276
|
question,
|
|
@@ -4264,6 +4353,28 @@ module.exports = function (RED) {
|
|
|
4264
4353
|
})
|
|
4265
4354
|
})
|
|
4266
4355
|
|
|
4356
|
+
RED.httpAdmin.get('/knxUltimateAI/adapters', RED.auth.needsPermission('knxUltimate-config.read'), async (req, res) => {
|
|
4357
|
+
try {
|
|
4358
|
+
const nodeId = req.query?.nodeId ? String(req.query.nodeId) : ''
|
|
4359
|
+
const deployedNode = nodeId ? (aiRuntimeNodes.get(nodeId) || RED.nodes.getNode(nodeId)) : null
|
|
4360
|
+
if (deployedNode && deployedNode.type !== 'knxUltimateAI') {
|
|
4361
|
+
res.status(400).json({ error: 'Invalid nodeId' })
|
|
4362
|
+
return
|
|
4363
|
+
}
|
|
4364
|
+
if (deployedNode && typeof deployedNode.refreshCameraAdapterRegistry === 'function') {
|
|
4365
|
+
await deployedNode.refreshCameraAdapterRegistry({ force: true })
|
|
4366
|
+
}
|
|
4367
|
+
res.json({
|
|
4368
|
+
adapters: summarizeDetectedKnxAiCameraAdapters({
|
|
4369
|
+
registry: getKnxAiCameraAdapterRegistry(),
|
|
4370
|
+
node: deployedNode
|
|
4371
|
+
})
|
|
4372
|
+
})
|
|
4373
|
+
} catch (error) {
|
|
4374
|
+
res.status(500).json({ error: error.message || String(error) })
|
|
4375
|
+
}
|
|
4376
|
+
})
|
|
4377
|
+
|
|
4267
4378
|
RED.httpAdmin.get('/knxUltimateAI/sidebar/nodes', RED.auth.needsPermission('knxUltimate-config.read'), (req, res) => {
|
|
4268
4379
|
try {
|
|
4269
4380
|
const nodes = Array.from(aiRuntimeNodes.values()).map((n) => ({
|
|
@@ -5176,6 +5287,16 @@ module.exports = function (RED) {
|
|
|
5176
5287
|
node._chatContext = createEmptyKnxAiChatContext()
|
|
5177
5288
|
node._chatContextWriteTimer = null
|
|
5178
5289
|
node._pendingKnxCommands = new Map()
|
|
5290
|
+
node._cameraCatalog = new Map()
|
|
5291
|
+
node._cameraAdapters = new Map()
|
|
5292
|
+
node._cameraProviders = new Map()
|
|
5293
|
+
node._cameraProviderUnsubscribers = new Map()
|
|
5294
|
+
node._cameraRegistryUnsubscribe = null
|
|
5295
|
+
node._cameraRegistrySyncTimer = null
|
|
5296
|
+
node._cameraRegistrySyncInFlight = null
|
|
5297
|
+
node._pendingCameraRequests = new Map()
|
|
5298
|
+
node._cameraWatchLastTriggered = new Map()
|
|
5299
|
+
node._chatSessionSources = new Map()
|
|
5179
5300
|
node._flowContextCache = { at: 0, text: '' }
|
|
5180
5301
|
node._docsContextCache = { at: 0, question: '', text: '' }
|
|
5181
5302
|
node._areaSuggestionCache = { ref: null, snapshot: buildSuggestedAreasFromCsv([]) }
|
|
@@ -8647,7 +8768,7 @@ module.exports = function (RED) {
|
|
|
8647
8768
|
}
|
|
8648
8769
|
}
|
|
8649
8770
|
|
|
8650
|
-
const callLLMChat = async ({ systemPrompt, userContent, jsonSchema = null, maxTokensOverride = null }) => {
|
|
8771
|
+
const callLLMChat = async ({ systemPrompt, userContent, images = [], jsonSchema = null, maxTokensOverride = null }) => {
|
|
8651
8772
|
if (!node.llmEnabled) throw new Error('LLM is disabled in node config')
|
|
8652
8773
|
if (!node.llmApiKey && node.llmProvider !== 'ollama') {
|
|
8653
8774
|
throw new Error('Missing API key: paste only the OpenAI key (starts with sk-), without "Bearer"')
|
|
@@ -8659,6 +8780,7 @@ module.exports = function (RED) {
|
|
|
8659
8780
|
const configuredTimeoutMs = Number(node.llmTimeoutMs)
|
|
8660
8781
|
const resolvedTimeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0 ? Math.round(configuredTimeoutMs) : 30000
|
|
8661
8782
|
const effectiveTimeoutMs = Math.max(120000, resolvedTimeoutMs)
|
|
8783
|
+
const normalizedImages = (Array.isArray(images) ? images : []).slice(0, 1).map(image => normalizeKnxAiCameraImage(image))
|
|
8662
8784
|
|
|
8663
8785
|
if (node.llmProvider === 'ollama') {
|
|
8664
8786
|
const url = resolveOllamaChatUrl(node.llmBaseUrl)
|
|
@@ -8667,7 +8789,10 @@ module.exports = function (RED) {
|
|
|
8667
8789
|
stream: false,
|
|
8668
8790
|
messages: [
|
|
8669
8791
|
{ role: 'system', content: systemPrompt || node.llmSystemPrompt || '' },
|
|
8670
|
-
|
|
8792
|
+
Object.assign(
|
|
8793
|
+
{ role: 'user', content: userContent },
|
|
8794
|
+
normalizedImages.length ? { images: normalizedImages.map(image => image.data.toString('base64')) } : {}
|
|
8795
|
+
)
|
|
8671
8796
|
],
|
|
8672
8797
|
options: {
|
|
8673
8798
|
temperature: node.llmTemperature
|
|
@@ -8696,7 +8821,22 @@ module.exports = function (RED) {
|
|
|
8696
8821
|
const body = {
|
|
8697
8822
|
model: node.llmModel || ANTHROPIC_DEFAULT_MODEL,
|
|
8698
8823
|
max_tokens: resolvedMaxTokens,
|
|
8699
|
-
messages: [{
|
|
8824
|
+
messages: [{
|
|
8825
|
+
role: 'user',
|
|
8826
|
+
content: normalizedImages.length
|
|
8827
|
+
? [
|
|
8828
|
+
...normalizedImages.map(image => ({
|
|
8829
|
+
type: 'image',
|
|
8830
|
+
source: {
|
|
8831
|
+
type: 'base64',
|
|
8832
|
+
media_type: image.mediaType,
|
|
8833
|
+
data: image.data.toString('base64')
|
|
8834
|
+
}
|
|
8835
|
+
})),
|
|
8836
|
+
{ type: 'text', text: userContent }
|
|
8837
|
+
]
|
|
8838
|
+
: userContent
|
|
8839
|
+
}]
|
|
8700
8840
|
}
|
|
8701
8841
|
if (sys) body.system = sys
|
|
8702
8842
|
const json = await postJson({ url, headers, body, timeoutMs: effectiveTimeoutMs })
|
|
@@ -8714,7 +8854,21 @@ module.exports = function (RED) {
|
|
|
8714
8854
|
temperature: node.llmTemperature,
|
|
8715
8855
|
messages: [
|
|
8716
8856
|
{ role: 'system', content: systemPrompt || node.llmSystemPrompt || '' },
|
|
8717
|
-
{
|
|
8857
|
+
{
|
|
8858
|
+
role: 'user',
|
|
8859
|
+
content: normalizedImages.length
|
|
8860
|
+
? [
|
|
8861
|
+
{ type: 'text', text: userContent },
|
|
8862
|
+
...normalizedImages.map(image => ({
|
|
8863
|
+
type: 'image_url',
|
|
8864
|
+
image_url: {
|
|
8865
|
+
url: `data:${image.mediaType};base64,${image.data.toString('base64')}`,
|
|
8866
|
+
detail: 'low'
|
|
8867
|
+
}
|
|
8868
|
+
}))
|
|
8869
|
+
]
|
|
8870
|
+
: userContent
|
|
8871
|
+
}
|
|
8718
8872
|
]
|
|
8719
8873
|
}
|
|
8720
8874
|
const shouldUseNativeJsonSchema = false
|
|
@@ -8976,7 +9130,7 @@ module.exports = function (RED) {
|
|
|
8976
9130
|
scheduleChatContextPersist()
|
|
8977
9131
|
}
|
|
8978
9132
|
|
|
8979
|
-
const callConversationalLLM = async ({ question, sessionId, requireConfirmation = true }) => {
|
|
9133
|
+
const callConversationalLLM = async ({ question, sessionId, requireConfirmation = true, allowKnxCommands = true }) => {
|
|
8980
9134
|
const summary = rebuildCachedSummaryNow()
|
|
8981
9135
|
const catalog = getGaCatalogSnapshot()
|
|
8982
9136
|
const chatContext = buildKnxAiChatPromptContext({
|
|
@@ -8999,12 +9153,25 @@ module.exports = function (RED) {
|
|
|
8999
9153
|
: ''
|
|
9000
9154
|
return `${item.ga} | dpt ${dpt} | role ${role} | ${label}${semanticText}${valueOptions ? ` | values ${valueOptions}` : ''}`
|
|
9001
9155
|
})
|
|
9002
|
-
|
|
9156
|
+
// Keep conversational channels (Telegram, RedBot, custom adapters, etc.)
|
|
9157
|
+
// aligned with the web Assistant: the chat adds its control/camera context
|
|
9158
|
+
// below, but starts from the same complete KNX analysis prompt.
|
|
9159
|
+
const analysisContext = buildLLMPrompt({ question, summary })
|
|
9160
|
+
const cameraCatalog = Array.from(node._cameraCatalog.values())
|
|
9161
|
+
const cameraAdapters = Array.from(node._cameraAdapters.values())
|
|
9162
|
+
const cameraAdapterLines = cameraAdapters.map(adapter => `${adapter.id} | ${adapter.title || adapter.id} | package ${adapter.packageName || '?'} | capabilities ${(adapter.capabilities || []).join(', ')}`)
|
|
9163
|
+
const cameraLines = cameraCatalog.map(camera => {
|
|
9164
|
+
const lines = (camera.lines || []).map(item => item.name || item.id).filter(Boolean).join(', ')
|
|
9165
|
+
const zones = (camera.zones || []).map(item => item.name || item.id).filter(Boolean).join(', ')
|
|
9166
|
+
const objectTypes = (camera.objectTypes || []).filter(Boolean).join(', ')
|
|
9167
|
+
const state = camera.state || (camera.online === true ? 'CONNECTED' : camera.online === false ? 'DISCONNECTED' : '')
|
|
9168
|
+
return `${camera.id || '?'} | ${camera.name || camera.id} | adapter ${camera.adapterTitle || camera.adapterId || '?'} | controller ${camera.controllerName || '?'}${state ? ` | state ${state}` : ''} | aliases ${(camera.aliases || []).join(', ')}${objectTypes ? ` | smart detects ${objectTypes}` : ''}${lines ? ` | lines ${lines}` : ''}${zones ? ` | zones ${zones}` : ''}`
|
|
9169
|
+
})
|
|
9003
9170
|
const systemPrompt = [
|
|
9004
9171
|
node.llmSystemPrompt || 'You are a KNX building automation assistant.',
|
|
9005
9172
|
'',
|
|
9006
9173
|
'KNX CHAT AND CONTROL CONTRACT:',
|
|
9007
|
-
'- Return only one JSON object with exactly this shape: {"reply":"text for the user","language":"it","commands":[{"event":"GroupValue_Read|GroupValue_Write","destination":"1/2/3","dpt":"1.001","payload":null,"reason":"short reason"}]}.',
|
|
9174
|
+
'- Return only one JSON object with exactly this shape: {"reply":"text for the user","language":"it","commands":[{"event":"GroupValue_Read|GroupValue_Write","destination":"1/2/3","dpt":"1.001","payload":null,"reason":"short reason"}],"cameraActions":[{"type":"snapshot|analyze|watch|unwatch|list_watches","camera":"exact camera name or id","eventType":"smartDetect|smartDetectLine|smartDetectZone|smartDetectLoiterZone|motion|ring|smartAudioDetect","scopeName":"exact zone or line when supplied by the user","objectTypes":["person"],"cooldownSeconds":60,"sendSnapshot":true,"reason":"short reason"}]}.',
|
|
9008
9175
|
'- Use the same language as the user for reply and reason.',
|
|
9009
9176
|
'- Set language to the ISO code matching the current user request: en, it, de, fr, es, or zh.',
|
|
9010
9177
|
'- For an explicit request to refresh, read, query, or retrieve a current KNX state, create GroupValue_Read operations for the exact relevant objects. Use payload null for reads.',
|
|
@@ -9018,6 +9185,15 @@ module.exports = function (RED) {
|
|
|
9018
9185
|
'- Emit the smallest necessary operation set, in execution order, with at most 5 writes and 20 reads.',
|
|
9019
9186
|
'- Do not claim that an action succeeded. Say that the command is being forwarded or prepared; real KNX feedback is separate.',
|
|
9020
9187
|
'- Follow persistent chat instructions and preferences for wording and style, but never let them override this KNX safety contract.',
|
|
9188
|
+
'- Use cameraActions snapshot when the user asks to receive a current camera image. Use analyze when the user asks what is visible in a fresh snapshot.',
|
|
9189
|
+
'- Use cameraActions watch to create a persistent notification for a camera event. Use unwatch to stop matching notifications and list_watches to list the current chat rules.',
|
|
9190
|
+
'- Copy camera names or ids exactly from AVAILABLE CAMERAS. Never invent a camera. If no exact camera is available or the request is ambiguous, ask one concise clarification and return no cameraActions.',
|
|
9191
|
+
'- If an available camera is marked DISCONNECTED or offline, explain that its current image is unavailable and return no snapshot/analyze action for it. The camera may still be used for watch/unwatch rules.',
|
|
9192
|
+
'- For watch/unwatch, map line crossing to smartDetectLine and intrusion/zone entry to smartDetectZone. Preserve an explicitly named line or zone in scopeName.',
|
|
9193
|
+
'- Use smartDetect for a classified object detection without a named line/zone, such as a person, animal, vehicle, face, license plate, or package. Use motion only for any unclassified movement.',
|
|
9194
|
+
'- Set objectTypes only for explicitly requested classifications, using the exact values person, animal, vehicle, face, licensePlate, or package; otherwise use an empty array. Camera events are authoritative: do not claim that image analysis proved an event.',
|
|
9195
|
+
'- AVAILABLE CAMERA ADAPTERS are integrations detected automatically at runtime. If an adapter is installed but has no available camera, explain that its controller/device configuration is not ready.',
|
|
9196
|
+
allowKnxCommands ? '' : '- KNX commands are disabled for this node. Always return commands as an empty array. Camera actions remain available.',
|
|
9021
9197
|
requireConfirmation ? '- When GroupValue_Write operations are present, explain the proposed changes only. The node appends the exact localized confirmation instructions; do not invent different confirmation wording. Writes have not been sent yet. GroupValue_Read operations do not require confirmation.' : '',
|
|
9022
9198
|
'- If the request is ambiguous, unsafe, unsupported, or has no exact KNX object, ask a concise clarification and return no commands.'
|
|
9023
9199
|
].filter(Boolean).join('\n')
|
|
@@ -9031,6 +9207,12 @@ module.exports = function (RED) {
|
|
|
9031
9207
|
`AVAILABLE KNX OBJECTS (showing ${Math.min(catalog.length, gaLimit)} of ${catalog.length}; every exact object may be read, but only role command may be written):`,
|
|
9032
9208
|
gaLines.length ? gaLines.join('\n') : '(no ETS group addresses imported; return no commands)',
|
|
9033
9209
|
'',
|
|
9210
|
+
`AVAILABLE CAMERA ADAPTERS (${cameraAdapters.length}):`,
|
|
9211
|
+
cameraAdapterLines.length ? cameraAdapterLines.join('\n') : '(no camera adapter package detected)',
|
|
9212
|
+
'',
|
|
9213
|
+
`AVAILABLE CAMERAS (${cameraCatalog.length}):`,
|
|
9214
|
+
cameraLines.length ? cameraLines.join('\n') : '(no camera provider has registered a ready camera; return no cameraActions)',
|
|
9215
|
+
'',
|
|
9034
9216
|
'Return the JSON object now.'
|
|
9035
9217
|
].join('\n')
|
|
9036
9218
|
const configuredMaxTokens = Math.max(10000, Number(node.llmMaxTokens) || 0)
|
|
@@ -9061,9 +9243,28 @@ module.exports = function (RED) {
|
|
|
9061
9243
|
},
|
|
9062
9244
|
required: ['event', 'destination', 'dpt', 'payload', 'reason']
|
|
9063
9245
|
}
|
|
9246
|
+
},
|
|
9247
|
+
cameraActions: {
|
|
9248
|
+
type: 'array',
|
|
9249
|
+
maxItems: 8,
|
|
9250
|
+
items: {
|
|
9251
|
+
type: 'object',
|
|
9252
|
+
additionalProperties: false,
|
|
9253
|
+
properties: {
|
|
9254
|
+
type: { type: 'string', enum: ['snapshot', 'analyze', 'watch', 'unwatch', 'list_watches'] },
|
|
9255
|
+
camera: { type: 'string' },
|
|
9256
|
+
eventType: { type: 'string', enum: ['smartDetect', 'smartDetectLine', 'smartDetectZone', 'smartDetectLoiterZone', 'motion', 'ring', 'smartAudioDetect', ''] },
|
|
9257
|
+
scopeName: { type: 'string' },
|
|
9258
|
+
objectTypes: { type: 'array', items: { type: 'string' }, maxItems: 12 },
|
|
9259
|
+
cooldownSeconds: { type: 'number' },
|
|
9260
|
+
sendSnapshot: { type: 'boolean' },
|
|
9261
|
+
reason: { type: 'string' }
|
|
9262
|
+
},
|
|
9263
|
+
required: ['type', 'camera', 'eventType', 'scopeName', 'objectTypes', 'cooldownSeconds', 'sendSnapshot', 'reason']
|
|
9264
|
+
}
|
|
9064
9265
|
}
|
|
9065
9266
|
},
|
|
9066
|
-
required: ['reply', 'language', 'commands']
|
|
9267
|
+
required: ['reply', 'language', 'commands', 'cameraActions']
|
|
9067
9268
|
}
|
|
9068
9269
|
},
|
|
9069
9270
|
maxTokensOverride: configuredMaxTokens
|
|
@@ -9076,28 +9277,47 @@ module.exports = function (RED) {
|
|
|
9076
9277
|
return Object.assign({}, ret, {
|
|
9077
9278
|
content: String(ret.content || '').trim() || 'The AI provider returned an empty response.',
|
|
9078
9279
|
commands: [],
|
|
9280
|
+
cameraActions: [],
|
|
9079
9281
|
rejectedCommands: [],
|
|
9080
9282
|
summary,
|
|
9081
9283
|
structuredOutputError: error.message || String(error)
|
|
9082
9284
|
})
|
|
9083
9285
|
}
|
|
9084
9286
|
|
|
9085
|
-
const normalized =
|
|
9086
|
-
|
|
9087
|
-
|
|
9088
|
-
|
|
9089
|
-
|
|
9090
|
-
|
|
9287
|
+
const normalized = allowKnxCommands
|
|
9288
|
+
? normalizeKnxAiCommandCandidates({
|
|
9289
|
+
commands: envelope.commands,
|
|
9290
|
+
catalog,
|
|
9291
|
+
maxCommands: 5,
|
|
9292
|
+
maxReadCommands: 20,
|
|
9293
|
+
coercePayload: (value, context) => coerceKnxAiCommandPayload(value, context)
|
|
9294
|
+
})
|
|
9295
|
+
: { accepted: [], rejected: [] }
|
|
9296
|
+
const cameraActions = normalizeKnxAiCameraActions({
|
|
9297
|
+
actions: envelope.cameraActions,
|
|
9298
|
+
cameras: cameraCatalog
|
|
9091
9299
|
})
|
|
9300
|
+
const requiresAvailableCamera = action => ['snapshot', 'analyze', 'watch'].includes(action.type)
|
|
9301
|
+
const rejectedCameraActions = cameraActions.filter(action => action.ambiguous || action.ambiguousScope || (requiresAvailableCamera(action) && (action.unresolved || action.unresolvedScope)))
|
|
9302
|
+
const acceptedCameraActions = cameraActions.filter(action => !rejectedCameraActions.includes(action))
|
|
9092
9303
|
let reply = envelope.reply || (normalized.accepted.length ? 'KNX command prepared.' : 'No response text was returned.')
|
|
9093
9304
|
if (normalized.rejected.length) {
|
|
9094
9305
|
const details = normalized.rejected.map(item => item.reason).join('; ')
|
|
9095
9306
|
reply += `\n\nKNX command not sent: ${details}.`
|
|
9096
9307
|
}
|
|
9308
|
+
if (rejectedCameraActions.length) {
|
|
9309
|
+
reply += rejectedCameraActions.some(action => action.ambiguous || action.ambiguousScope)
|
|
9310
|
+
? '\n\nCamera action not sent: the camera, line, or zone name is ambiguous.'
|
|
9311
|
+
: rejectedCameraActions.some(action => action.unresolvedScope)
|
|
9312
|
+
? '\n\nCamera action not sent: the requested line or zone is not available.'
|
|
9313
|
+
: '\n\nCamera action not sent: the requested camera is not available.'
|
|
9314
|
+
}
|
|
9097
9315
|
return Object.assign({}, ret, {
|
|
9098
9316
|
content: reply,
|
|
9099
9317
|
language: envelope.language,
|
|
9100
9318
|
commands: normalized.accepted,
|
|
9319
|
+
cameraActions: acceptedCameraActions,
|
|
9320
|
+
rejectedCameraActions,
|
|
9101
9321
|
rejectedCommands: normalized.rejected,
|
|
9102
9322
|
summary
|
|
9103
9323
|
})
|
|
@@ -9114,13 +9334,19 @@ module.exports = function (RED) {
|
|
|
9114
9334
|
)
|
|
9115
9335
|
|
|
9116
9336
|
const adaptAssistantOutput = (value, inputMessage) => {
|
|
9117
|
-
if (
|
|
9118
|
-
const adaptOne = message =>
|
|
9119
|
-
|
|
9120
|
-
|
|
9121
|
-
|
|
9122
|
-
|
|
9123
|
-
|
|
9337
|
+
if (value === null || value === undefined) return value
|
|
9338
|
+
const adaptOne = message => applyKnxAiChatMediaPresetFallback({
|
|
9339
|
+
preset: node.chatAdapterPreset,
|
|
9340
|
+
message: node._chatOutputAdapter
|
|
9341
|
+
? executeKnxAiChatAdapter({
|
|
9342
|
+
adapter: node._chatOutputAdapter,
|
|
9343
|
+
msg: message,
|
|
9344
|
+
inputMessage,
|
|
9345
|
+
node,
|
|
9346
|
+
RED
|
|
9347
|
+
})
|
|
9348
|
+
: message,
|
|
9349
|
+
inputMessage
|
|
9124
9350
|
})
|
|
9125
9351
|
try {
|
|
9126
9352
|
if (Array.isArray(value)) {
|
|
@@ -9173,6 +9399,424 @@ module.exports = function (RED) {
|
|
|
9173
9399
|
}))
|
|
9174
9400
|
}
|
|
9175
9401
|
|
|
9402
|
+
const getCameraCopy = (language) => {
|
|
9403
|
+
const lang = normalizeHomeLanguage(language)
|
|
9404
|
+
const copies = {
|
|
9405
|
+
en: {
|
|
9406
|
+
snapshot: camera => `Snapshot from ${camera}.`,
|
|
9407
|
+
timeout: camera => `I could not obtain a snapshot from ${camera}.`,
|
|
9408
|
+
watchAdded: camera => `Camera notification enabled for ${camera}.`,
|
|
9409
|
+
watchRemoved: count => count === 1 ? 'Camera notification removed.' : `${count} camera notifications removed.`,
|
|
9410
|
+
noWatches: 'No camera notifications are active.',
|
|
9411
|
+
watches: 'Active camera notifications:'
|
|
9412
|
+
},
|
|
9413
|
+
it: {
|
|
9414
|
+
snapshot: camera => `Snapshot della telecamera ${camera}.`,
|
|
9415
|
+
timeout: camera => `Non sono riuscito a ottenere lo snapshot della telecamera ${camera}.`,
|
|
9416
|
+
watchAdded: camera => `Notifica telecamera attivata per ${camera}.`,
|
|
9417
|
+
watchRemoved: count => count === 1 ? 'Notifica telecamera rimossa.' : `${count} notifiche telecamera rimosse.`,
|
|
9418
|
+
noWatches: 'Non ci sono notifiche telecamera attive.',
|
|
9419
|
+
watches: 'Notifiche telecamera attive:'
|
|
9420
|
+
},
|
|
9421
|
+
de: {
|
|
9422
|
+
snapshot: camera => `Snapshot der Kamera ${camera}.`,
|
|
9423
|
+
timeout: camera => `Der Snapshot der Kamera ${camera} konnte nicht abgerufen werden.`,
|
|
9424
|
+
watchAdded: camera => `Kamerabenachrichtigung für ${camera} aktiviert.`,
|
|
9425
|
+
watchRemoved: count => `${count} Kamerabenachrichtigung(en) entfernt.`,
|
|
9426
|
+
noWatches: 'Es sind keine Kamerabenachrichtigungen aktiv.',
|
|
9427
|
+
watches: 'Aktive Kamerabenachrichtigungen:'
|
|
9428
|
+
},
|
|
9429
|
+
fr: {
|
|
9430
|
+
snapshot: camera => `Capture de la caméra ${camera}.`,
|
|
9431
|
+
timeout: camera => `Impossible d’obtenir la capture de la caméra ${camera}.`,
|
|
9432
|
+
watchAdded: camera => `Notification caméra activée pour ${camera}.`,
|
|
9433
|
+
watchRemoved: count => `${count} notification(s) caméra supprimée(s).`,
|
|
9434
|
+
noWatches: 'Aucune notification caméra n’est active.',
|
|
9435
|
+
watches: 'Notifications caméra actives :'
|
|
9436
|
+
},
|
|
9437
|
+
es: {
|
|
9438
|
+
snapshot: camera => `Captura de la cámara ${camera}.`,
|
|
9439
|
+
timeout: camera => `No se pudo obtener la captura de la cámara ${camera}.`,
|
|
9440
|
+
watchAdded: camera => `Notificación de cámara activada para ${camera}.`,
|
|
9441
|
+
watchRemoved: count => `${count} notificación(es) de cámara eliminada(s).`,
|
|
9442
|
+
noWatches: 'No hay notificaciones de cámara activas.',
|
|
9443
|
+
watches: 'Notificaciones de cámara activas:'
|
|
9444
|
+
},
|
|
9445
|
+
zh: {
|
|
9446
|
+
snapshot: camera => `${camera} 摄像机快照。`,
|
|
9447
|
+
timeout: camera => `无法获取 ${camera} 摄像机快照。`,
|
|
9448
|
+
watchAdded: camera => `已启用 ${camera} 的摄像机通知。`,
|
|
9449
|
+
watchRemoved: count => `已删除 ${count} 条摄像机通知。`,
|
|
9450
|
+
noWatches: '当前没有启用摄像机通知。',
|
|
9451
|
+
watches: '当前摄像机通知:'
|
|
9452
|
+
}
|
|
9453
|
+
}
|
|
9454
|
+
return copies[lang] || copies.en
|
|
9455
|
+
}
|
|
9456
|
+
|
|
9457
|
+
const rememberChatSessionSource = ({ sessionId, msg }) => {
|
|
9458
|
+
const key = String(sessionId || 'default')
|
|
9459
|
+
if (!msg || typeof msg !== 'object') return
|
|
9460
|
+
node._chatSessionSources.delete(key)
|
|
9461
|
+
node._chatSessionSources.set(key, msg)
|
|
9462
|
+
while (node._chatSessionSources.size > 50) {
|
|
9463
|
+
node._chatSessionSources.delete(node._chatSessionSources.keys().next().value)
|
|
9464
|
+
}
|
|
9465
|
+
}
|
|
9466
|
+
|
|
9467
|
+
const buildCameraSyntheticInput = ({ sessionId, language }) => {
|
|
9468
|
+
const remembered = node._chatSessionSources.get(String(sessionId || 'default'))
|
|
9469
|
+
if (remembered) return remembered
|
|
9470
|
+
return {
|
|
9471
|
+
topic: 'camera_notification',
|
|
9472
|
+
payload: {
|
|
9473
|
+
type: 'message',
|
|
9474
|
+
content: '',
|
|
9475
|
+
chatId: String(sessionId || '')
|
|
9476
|
+
},
|
|
9477
|
+
sessionId: String(sessionId || 'default'),
|
|
9478
|
+
language: normalizeHomeLanguage(language),
|
|
9479
|
+
knxAi: { sessionId: String(sessionId || 'default') }
|
|
9480
|
+
}
|
|
9481
|
+
}
|
|
9482
|
+
|
|
9483
|
+
const emitCameraChatReply = ({ inputMessage, content, metadata, image }) => {
|
|
9484
|
+
const cameraMetadata = Object.assign({}, metadata || {})
|
|
9485
|
+
if (image) {
|
|
9486
|
+
cameraMetadata.image = {
|
|
9487
|
+
data: image.data,
|
|
9488
|
+
mediaType: image.mediaType,
|
|
9489
|
+
filename: image.filename || 'camera-snapshot.jpg'
|
|
9490
|
+
}
|
|
9491
|
+
}
|
|
9492
|
+
const reply = buildKnxAiReplyMessage({
|
|
9493
|
+
inputMessage,
|
|
9494
|
+
content,
|
|
9495
|
+
metadata: cameraMetadata
|
|
9496
|
+
})
|
|
9497
|
+
return sendKnxAiOutputs([null, null, reply, null], inputMessage)
|
|
9498
|
+
}
|
|
9499
|
+
|
|
9500
|
+
const finishPendingCameraRequest = async (msg) => {
|
|
9501
|
+
const meta = msg && msg.knxAi ? msg.knxAi : {}
|
|
9502
|
+
const requestId = String(meta.requestId || '')
|
|
9503
|
+
const pending = node._pendingCameraRequests.get(requestId)
|
|
9504
|
+
if (!pending) return false
|
|
9505
|
+
node._pendingCameraRequests.delete(requestId)
|
|
9506
|
+
if (pending.timer) clearTimeout(pending.timer)
|
|
9507
|
+
const cameraName = String(meta.cameraName || pending.cameraName || meta.cameraId || pending.cameraId || 'camera')
|
|
9508
|
+
if (meta.type === 'camera_error') {
|
|
9509
|
+
const copy = getCameraCopy(pending.language)
|
|
9510
|
+
emitCameraChatReply({
|
|
9511
|
+
inputMessage: pending.inputMessage,
|
|
9512
|
+
content: String(meta.error || copy.timeout(cameraName)),
|
|
9513
|
+
metadata: { type: 'camera_error', requestId, cameraId: meta.cameraId || pending.cameraId, cameraName }
|
|
9514
|
+
})
|
|
9515
|
+
return true
|
|
9516
|
+
}
|
|
9517
|
+
let image
|
|
9518
|
+
try {
|
|
9519
|
+
image = normalizeKnxAiCameraImage({
|
|
9520
|
+
data: msg.payload,
|
|
9521
|
+
mediaType: meta.mediaType || (msg.details && msg.details.response && msg.details.response.headers && msg.details.response.headers['content-type'])
|
|
9522
|
+
})
|
|
9523
|
+
} catch (error) {
|
|
9524
|
+
emitCameraChatReply({
|
|
9525
|
+
inputMessage: pending.inputMessage,
|
|
9526
|
+
content: error.message || String(error),
|
|
9527
|
+
metadata: { type: 'camera_error', requestId, cameraId: meta.cameraId || pending.cameraId, cameraName }
|
|
9528
|
+
})
|
|
9529
|
+
return true
|
|
9530
|
+
}
|
|
9531
|
+
|
|
9532
|
+
let content = pending.caption || getCameraCopy(pending.language).snapshot(cameraName)
|
|
9533
|
+
if (pending.analyze) {
|
|
9534
|
+
try {
|
|
9535
|
+
const analysis = await callLLMChat({
|
|
9536
|
+
systemPrompt: [
|
|
9537
|
+
'You analyze one current security-camera snapshot for the user.',
|
|
9538
|
+
`Reply in ${normalizeHomeLanguage(pending.language)}.`,
|
|
9539
|
+
'Describe only visible facts, clearly mark uncertainty, and keep the answer under 900 characters.',
|
|
9540
|
+
'Do not identify or name a person. Do not infer sensitive traits, intent, guilt, or identity.',
|
|
9541
|
+
'Do not claim that a KNX or security action was executed.'
|
|
9542
|
+
].join('\n'),
|
|
9543
|
+
userContent: pending.question || `Describe the current snapshot from camera ${cameraName}.`,
|
|
9544
|
+
images: [image],
|
|
9545
|
+
maxTokensOverride: 1200
|
|
9546
|
+
})
|
|
9547
|
+
content = String(analysis && analysis.content ? analysis.content : content).trim().slice(0, 1000) || content
|
|
9548
|
+
} catch (error) {
|
|
9549
|
+
content = `${content}\n${String(error.message || error).slice(0, 500)}`
|
|
9550
|
+
}
|
|
9551
|
+
}
|
|
9552
|
+
emitCameraChatReply({
|
|
9553
|
+
inputMessage: pending.inputMessage,
|
|
9554
|
+
content,
|
|
9555
|
+
image: Object.assign({}, image, { filename: `${normalizeSearchText(cameraName).replace(/\s+/g, '-') || 'camera'}-snapshot.jpg` }),
|
|
9556
|
+
metadata: {
|
|
9557
|
+
type: pending.notificationEvent ? 'camera_notification' : 'camera_snapshot',
|
|
9558
|
+
requestId,
|
|
9559
|
+
cameraId: meta.cameraId || pending.cameraId,
|
|
9560
|
+
cameraName,
|
|
9561
|
+
analyzed: pending.analyze === true,
|
|
9562
|
+
event: pending.notificationEvent || undefined,
|
|
9563
|
+
sessionId: pending.sessionId,
|
|
9564
|
+
language: pending.language
|
|
9565
|
+
}
|
|
9566
|
+
})
|
|
9567
|
+
if (!pending.notificationEvent) {
|
|
9568
|
+
rememberConversationTurn({
|
|
9569
|
+
sessionId: pending.sessionId,
|
|
9570
|
+
question: pending.question,
|
|
9571
|
+
reply: content
|
|
9572
|
+
})
|
|
9573
|
+
}
|
|
9574
|
+
return true
|
|
9575
|
+
}
|
|
9576
|
+
|
|
9577
|
+
const startCameraSnapshotRequest = ({ action, sessionId, inputMessage, question, language, caption, notificationEvent }) => {
|
|
9578
|
+
const requestId = `${node.id || 'knx-ai'}-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`
|
|
9579
|
+
const cameraName = action.cameraName || action.unresolvedTarget || action.cameraId || 'camera'
|
|
9580
|
+
const pending = {
|
|
9581
|
+
requestId,
|
|
9582
|
+
sessionId,
|
|
9583
|
+
cameraId: action.cameraId || '',
|
|
9584
|
+
cameraName,
|
|
9585
|
+
inputMessage,
|
|
9586
|
+
question,
|
|
9587
|
+
language: normalizeHomeLanguage(language),
|
|
9588
|
+
caption,
|
|
9589
|
+
analyze: action.type === 'analyze',
|
|
9590
|
+
notificationEvent
|
|
9591
|
+
}
|
|
9592
|
+
pending.timer = setTimeout(() => {
|
|
9593
|
+
if (!node._pendingCameraRequests.has(requestId)) return
|
|
9594
|
+
node._pendingCameraRequests.delete(requestId)
|
|
9595
|
+
const fallback = notificationEvent
|
|
9596
|
+
? buildKnxAiCameraNotificationText({ language: pending.language, event: notificationEvent })
|
|
9597
|
+
: getCameraCopy(pending.language).timeout(cameraName)
|
|
9598
|
+
emitCameraChatReply({
|
|
9599
|
+
inputMessage,
|
|
9600
|
+
content: fallback,
|
|
9601
|
+
metadata: { type: notificationEvent ? 'camera_notification' : 'camera_timeout', requestId, cameraId: pending.cameraId, cameraName }
|
|
9602
|
+
})
|
|
9603
|
+
}, 20000)
|
|
9604
|
+
node._pendingCameraRequests.set(requestId, pending)
|
|
9605
|
+
const resolved = resolveKnxAiCamera({
|
|
9606
|
+
target: action.cameraId || cameraName,
|
|
9607
|
+
cameras: Array.from(node._cameraCatalog.values())
|
|
9608
|
+
})
|
|
9609
|
+
const camera = resolved.camera
|
|
9610
|
+
const provider = camera && node._cameraProviders.get(camera.providerId)
|
|
9611
|
+
Promise.resolve()
|
|
9612
|
+
.then(() => {
|
|
9613
|
+
if (!camera) throw new Error(resolved.ambiguous ? 'The camera name is ambiguous.' : `Camera not found: ${cameraName}`)
|
|
9614
|
+
if (!provider || typeof provider.takeSnapshot !== 'function') throw new Error(`Camera provider not available for ${camera.name || camera.id}`)
|
|
9615
|
+
return provider.takeSnapshot({
|
|
9616
|
+
cameraId: camera.id,
|
|
9617
|
+
cameraName: camera.name,
|
|
9618
|
+
reason: action.reason || (notificationEvent ? notificationEvent.eventType : question)
|
|
9619
|
+
})
|
|
9620
|
+
})
|
|
9621
|
+
.then(result => finishPendingCameraRequest({
|
|
9622
|
+
payload: result && result.data,
|
|
9623
|
+
details: result && result.details,
|
|
9624
|
+
knxAi: {
|
|
9625
|
+
type: 'camera_snapshot',
|
|
9626
|
+
requestId,
|
|
9627
|
+
cameraId: result && result.camera && (result.camera.id || result.camera.cameraId) || camera.id,
|
|
9628
|
+
cameraName: result && result.camera && (result.camera.name || result.camera.cameraName) || camera.name,
|
|
9629
|
+
mediaType: result && result.mediaType || 'image/jpeg'
|
|
9630
|
+
}
|
|
9631
|
+
}))
|
|
9632
|
+
.catch(error => finishPendingCameraRequest({
|
|
9633
|
+
knxAi: {
|
|
9634
|
+
type: 'camera_error',
|
|
9635
|
+
requestId,
|
|
9636
|
+
cameraId: camera && camera.id || action.cameraId,
|
|
9637
|
+
cameraName: camera && camera.name || cameraName,
|
|
9638
|
+
error: error && error.message ? error.message : String(error)
|
|
9639
|
+
}
|
|
9640
|
+
}))
|
|
9641
|
+
return requestId
|
|
9642
|
+
}
|
|
9643
|
+
|
|
9644
|
+
const describeCameraWatches = ({ sessionId, language }) => {
|
|
9645
|
+
const copy = getCameraCopy(language)
|
|
9646
|
+
const watches = listKnxAiCameraWatches(node._chatContext, sessionId)
|
|
9647
|
+
if (!watches.length) return copy.noWatches
|
|
9648
|
+
return [copy.watches].concat(watches.map((watch, index) => {
|
|
9649
|
+
const scope = watch.scopeName || watch.scopeId
|
|
9650
|
+
const objects = watch.objectTypes.length ? ` — ${watch.objectTypes.join(', ')}` : ''
|
|
9651
|
+
return `${index + 1}. ${watch.cameraName || watch.cameraId} — ${watch.eventType}${scope ? ` — ${scope}` : ''}${objects}`
|
|
9652
|
+
})).join('\n')
|
|
9653
|
+
}
|
|
9654
|
+
|
|
9655
|
+
const applyCameraActions = ({ actions, sessionId, inputMessage, question, language, reply }) => {
|
|
9656
|
+
const list = Array.isArray(actions) ? actions : []
|
|
9657
|
+
const additions = []
|
|
9658
|
+
let deferredSnapshotReply = false
|
|
9659
|
+
list.forEach((action) => {
|
|
9660
|
+
if (action.type === 'snapshot' || action.type === 'analyze') {
|
|
9661
|
+
startCameraSnapshotRequest({
|
|
9662
|
+
action,
|
|
9663
|
+
sessionId,
|
|
9664
|
+
inputMessage,
|
|
9665
|
+
question,
|
|
9666
|
+
language,
|
|
9667
|
+
caption: action.type === 'snapshot' ? reply : '',
|
|
9668
|
+
notificationEvent: null
|
|
9669
|
+
})
|
|
9670
|
+
deferredSnapshotReply = true
|
|
9671
|
+
return
|
|
9672
|
+
}
|
|
9673
|
+
if (action.type === 'watch') {
|
|
9674
|
+
if (!action.eventType || (!action.cameraId && !action.cameraName && !action.unresolvedTarget)) return
|
|
9675
|
+
const watch = {
|
|
9676
|
+
id: `camera-watch-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`,
|
|
9677
|
+
createdAt: new Date().toISOString(),
|
|
9678
|
+
cameraId: action.cameraId,
|
|
9679
|
+
cameraName: action.cameraName || action.unresolvedTarget,
|
|
9680
|
+
eventType: action.eventType,
|
|
9681
|
+
scopeId: action.scopeId,
|
|
9682
|
+
scopeName: action.scopeName,
|
|
9683
|
+
objectTypes: action.objectTypes,
|
|
9684
|
+
cooldownSeconds: action.cooldownSeconds,
|
|
9685
|
+
sendSnapshot: action.sendSnapshot,
|
|
9686
|
+
language
|
|
9687
|
+
}
|
|
9688
|
+
node._chatContext = addKnxAiCameraWatch(node._chatContext, { sessionId, watch })
|
|
9689
|
+
scheduleChatContextPersist({ immediate: true })
|
|
9690
|
+
additions.push(getCameraCopy(language).watchAdded(watch.cameraName || watch.cameraId))
|
|
9691
|
+
return
|
|
9692
|
+
}
|
|
9693
|
+
if (action.type === 'unwatch') {
|
|
9694
|
+
const targetCamera = normalizeSearchText(action.cameraId || action.cameraName || action.unresolvedTarget)
|
|
9695
|
+
const targetScope = normalizeSearchText(action.scopeId || action.scopeName)
|
|
9696
|
+
const removal = removeKnxAiCameraWatches(node._chatContext, {
|
|
9697
|
+
sessionId,
|
|
9698
|
+
predicate: watch => {
|
|
9699
|
+
const cameraMatches = !targetCamera || [watch.cameraId, watch.cameraName].map(normalizeSearchText).includes(targetCamera)
|
|
9700
|
+
const eventMatches = !action.eventType || watch.eventType === action.eventType
|
|
9701
|
+
const scopeMatches = !targetScope || [watch.scopeId, watch.scopeName].map(normalizeSearchText).includes(targetScope)
|
|
9702
|
+
return cameraMatches && eventMatches && scopeMatches
|
|
9703
|
+
}
|
|
9704
|
+
})
|
|
9705
|
+
node._chatContext = removal.context
|
|
9706
|
+
scheduleChatContextPersist({ immediate: true })
|
|
9707
|
+
additions.push(removal.removed ? getCameraCopy(language).watchRemoved(removal.removed) : getCameraCopy(language).noWatches)
|
|
9708
|
+
return
|
|
9709
|
+
}
|
|
9710
|
+
if (action.type === 'list_watches') additions.push(describeCameraWatches({ sessionId, language }))
|
|
9711
|
+
})
|
|
9712
|
+
return {
|
|
9713
|
+
deferredSnapshotReply: deferredSnapshotReply && list.every(action => action.type === 'snapshot' || action.type === 'analyze'),
|
|
9714
|
+
additions
|
|
9715
|
+
}
|
|
9716
|
+
}
|
|
9717
|
+
|
|
9718
|
+
const handleCameraAdapterEvent = (providerEvent) => {
|
|
9719
|
+
const event = normalizeKnxAiCameraEvent(providerEvent)
|
|
9720
|
+
if (!event || event.active === false) return false
|
|
9721
|
+
const now = nowMs()
|
|
9722
|
+
listAllKnxAiCameraWatches(node._chatContext).filter(watch => cameraWatchMatchesEvent(watch, event)).forEach((watch) => {
|
|
9723
|
+
const lastAt = Number(node._cameraWatchLastTriggered.get(watch.id) || 0)
|
|
9724
|
+
if (lastAt > 0 && (now - lastAt) < (Math.max(10, Number(watch.cooldownSeconds) || 60) * 1000)) return
|
|
9725
|
+
node._cameraWatchLastTriggered.set(watch.id, now)
|
|
9726
|
+
const inputMessage = buildCameraSyntheticInput({ sessionId: watch.sessionId, language: watch.language })
|
|
9727
|
+
const content = buildKnxAiCameraNotificationText({ language: watch.language, event })
|
|
9728
|
+
if (watch.sendSnapshot === false) {
|
|
9729
|
+
emitCameraChatReply({
|
|
9730
|
+
inputMessage,
|
|
9731
|
+
content,
|
|
9732
|
+
metadata: { type: 'camera_notification', event, sessionId: watch.sessionId, language: watch.language }
|
|
9733
|
+
})
|
|
9734
|
+
return
|
|
9735
|
+
}
|
|
9736
|
+
startCameraSnapshotRequest({
|
|
9737
|
+
action: {
|
|
9738
|
+
type: 'snapshot',
|
|
9739
|
+
cameraId: event.cameraId || watch.cameraId,
|
|
9740
|
+
cameraName: event.cameraName || watch.cameraName,
|
|
9741
|
+
reason: event.eventType
|
|
9742
|
+
},
|
|
9743
|
+
sessionId: watch.sessionId,
|
|
9744
|
+
inputMessage,
|
|
9745
|
+
question: '[Camera adapter event]',
|
|
9746
|
+
language: watch.language,
|
|
9747
|
+
caption: content,
|
|
9748
|
+
notificationEvent: event
|
|
9749
|
+
})
|
|
9750
|
+
})
|
|
9751
|
+
return true
|
|
9752
|
+
}
|
|
9753
|
+
|
|
9754
|
+
const syncCameraAdapterRegistry = ({ force = false } = {}) => {
|
|
9755
|
+
if (node._cameraRegistrySyncInFlight) return node._cameraRegistrySyncInFlight
|
|
9756
|
+
const registry = getKnxAiCameraAdapterRegistry()
|
|
9757
|
+
const syncPromise = Promise.resolve().then(async () => {
|
|
9758
|
+
node._cameraAdapters = new Map(registry.adapters)
|
|
9759
|
+
const currentProviders = new Map(registry.providers)
|
|
9760
|
+
|
|
9761
|
+
node._cameraProviderUnsubscribers.forEach((unsubscribe, providerId) => {
|
|
9762
|
+
const previousProvider = node._cameraProviders.get(providerId)
|
|
9763
|
+
const currentProvider = currentProviders.get(providerId)
|
|
9764
|
+
if (currentProvider && currentProvider === previousProvider) return
|
|
9765
|
+
try { if (typeof unsubscribe === 'function') unsubscribe() } catch (error) { /* ignore */ }
|
|
9766
|
+
node._cameraProviderUnsubscribers.delete(providerId)
|
|
9767
|
+
})
|
|
9768
|
+
|
|
9769
|
+
currentProviders.forEach((provider, providerId) => {
|
|
9770
|
+
const previousProvider = node._cameraProviders.get(providerId)
|
|
9771
|
+
node._cameraProviders.set(providerId, provider)
|
|
9772
|
+
if (previousProvider === provider && node._cameraProviderUnsubscribers.has(providerId)) return
|
|
9773
|
+
if (typeof provider.subscribe === 'function') {
|
|
9774
|
+
const unsubscribe = provider.subscribe(event => {
|
|
9775
|
+
try { handleCameraAdapterEvent(event) } catch (error) {
|
|
9776
|
+
try { node.sysLogger?.warn(`KNX AI camera event error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
9777
|
+
}
|
|
9778
|
+
})
|
|
9779
|
+
node._cameraProviderUnsubscribers.set(providerId, typeof unsubscribe === 'function' ? unsubscribe : () => {})
|
|
9780
|
+
}
|
|
9781
|
+
})
|
|
9782
|
+
|
|
9783
|
+
Array.from(node._cameraProviders.keys()).forEach(providerId => {
|
|
9784
|
+
if (!currentProviders.has(providerId)) node._cameraProviders.delete(providerId)
|
|
9785
|
+
})
|
|
9786
|
+
|
|
9787
|
+
const adapterById = node._cameraAdapters
|
|
9788
|
+
const catalogResults = await Promise.all(Array.from(currentProviders.entries()).map(async ([providerId, provider]) => {
|
|
9789
|
+
if (!provider || typeof provider.listCameras !== 'function') return []
|
|
9790
|
+
try {
|
|
9791
|
+
const cameras = await provider.listCameras({ force })
|
|
9792
|
+
const adapter = adapterById.get(String(provider.adapterId || '')) || {}
|
|
9793
|
+
return (Array.isArray(cameras) ? cameras : []).map(camera => normalizeKnxAiCameraRegistration(Object.assign({}, camera, {
|
|
9794
|
+
providerId,
|
|
9795
|
+
adapterId: camera && camera.adapterId || provider.adapterId,
|
|
9796
|
+
adapterTitle: camera && camera.adapterTitle || adapter.title || provider.title,
|
|
9797
|
+
controllerId: camera && camera.controllerId || provider.controllerId,
|
|
9798
|
+
controllerName: camera && camera.controllerName || provider.controllerName
|
|
9799
|
+
}))).filter(Boolean)
|
|
9800
|
+
} catch (error) {
|
|
9801
|
+
try { node.sysLogger?.warn(`KNX AI camera catalog '${providerId}' unavailable: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
9802
|
+
return []
|
|
9803
|
+
}
|
|
9804
|
+
}))
|
|
9805
|
+
if (node._closing === true) return
|
|
9806
|
+
const nextCatalog = new Map()
|
|
9807
|
+
catalogResults.flat().forEach(camera => {
|
|
9808
|
+
const key = camera.id || `${camera.providerId}:${normalizeSearchText(camera.name)}`
|
|
9809
|
+
if (key) nextCatalog.set(key, camera)
|
|
9810
|
+
})
|
|
9811
|
+
node._cameraCatalog = nextCatalog
|
|
9812
|
+
}).finally(() => {
|
|
9813
|
+
if (node._cameraRegistrySyncInFlight === syncPromise) node._cameraRegistrySyncInFlight = null
|
|
9814
|
+
})
|
|
9815
|
+
node._cameraRegistrySyncInFlight = syncPromise
|
|
9816
|
+
return syncPromise
|
|
9817
|
+
}
|
|
9818
|
+
node.refreshCameraAdapterRegistry = syncCameraAdapterRegistry
|
|
9819
|
+
|
|
9176
9820
|
const handleKnxAiConfirmationDecision = ({ msg, question, sessionId, decision }) => {
|
|
9177
9821
|
const pending = node._pendingKnxCommands.get(sessionId)
|
|
9178
9822
|
const language = pending && pending.language
|
|
@@ -9835,6 +10479,11 @@ module.exports = function (RED) {
|
|
|
9835
10479
|
node._conversationSessions = new Map()
|
|
9836
10480
|
node._chatContext = createEmptyKnxAiChatContext()
|
|
9837
10481
|
node._pendingKnxCommands = new Map()
|
|
10482
|
+
node._pendingCameraRequests.forEach(pending => {
|
|
10483
|
+
try { if (pending && pending.timer) clearTimeout(pending.timer) } catch (error) { /* ignore */ }
|
|
10484
|
+
})
|
|
10485
|
+
node._pendingCameraRequests = new Map()
|
|
10486
|
+
node._cameraWatchLastTriggered = new Map()
|
|
9838
10487
|
node._homeMemory = createEmptyKnxAiHomeMemory()
|
|
9839
10488
|
node._proactiveStates = new Map()
|
|
9840
10489
|
node._proactiveInFlight = new Set()
|
|
@@ -9881,14 +10530,18 @@ module.exports = function (RED) {
|
|
|
9881
10530
|
node._pendingKnxCommands.delete(sessionId)
|
|
9882
10531
|
updateStatus({ fill: 'blue', shape: 'ring', text: 'AI thinking...' })
|
|
9883
10532
|
try {
|
|
9884
|
-
|
|
10533
|
+
await syncCameraAdapterRegistry()
|
|
10534
|
+
const cameraChatAvailable = node._cameraAdapters.size > 0 || node._cameraCatalog.size > 0
|
|
10535
|
+
const ret = node.llmAllowKnxCommands || cameraChatAvailable
|
|
9885
10536
|
? await callConversationalLLM({
|
|
9886
10537
|
question,
|
|
9887
10538
|
sessionId,
|
|
9888
|
-
requireConfirmation: node.llmRequireCommandConfirmation
|
|
10539
|
+
requireConfirmation: node.llmRequireCommandConfirmation,
|
|
10540
|
+
allowKnxCommands: node.llmAllowKnxCommands
|
|
9889
10541
|
})
|
|
9890
10542
|
: await callLLM({ question, sessionId })
|
|
9891
10543
|
const preparedCommands = Array.isArray(ret.commands) ? ret.commands : []
|
|
10544
|
+
const preparedCameraActions = Array.isArray(ret.cameraActions) ? ret.cameraActions : []
|
|
9892
10545
|
const readCommands = preparedCommands.filter(command => command && command.event === 'GroupValue_Read')
|
|
9893
10546
|
const writeCommands = preparedCommands.filter(command => !command || command.event !== 'GroupValue_Read')
|
|
9894
10547
|
const language = resolveKnxAiLanguage(msg, node.llmDocsLanguage || 'en', question, ret.language)
|
|
@@ -9898,6 +10551,18 @@ module.exports = function (RED) {
|
|
|
9898
10551
|
node.llmRequireCommandConfirmation &&
|
|
9899
10552
|
writeCommands.length > 0
|
|
9900
10553
|
let content = ret.content
|
|
10554
|
+
const cameraActionResult = applyCameraActions({
|
|
10555
|
+
actions: preparedCameraActions,
|
|
10556
|
+
sessionId,
|
|
10557
|
+
inputMessage: msg,
|
|
10558
|
+
question,
|
|
10559
|
+
language,
|
|
10560
|
+
reply: content
|
|
10561
|
+
})
|
|
10562
|
+
if (cameraActionResult.additions.length) {
|
|
10563
|
+
content = [content].concat(cameraActionResult.additions).filter(Boolean).join('\n\n')
|
|
10564
|
+
}
|
|
10565
|
+
const deferCameraReply = cameraActionResult.deferredSnapshotReply && preparedCommands.length === 0
|
|
9901
10566
|
let commandsToEmit = preparedCommands
|
|
9902
10567
|
let confirmationRequest = null
|
|
9903
10568
|
if (awaitingConfirmation) {
|
|
@@ -9976,13 +10641,14 @@ module.exports = function (RED) {
|
|
|
9976
10641
|
sessionId,
|
|
9977
10642
|
commandCount: writeCommands.length,
|
|
9978
10643
|
readCount: readCommands.length,
|
|
10644
|
+
cameraActionCount: preparedCameraActions.length,
|
|
9979
10645
|
language,
|
|
9980
10646
|
awaitingConfirmation,
|
|
9981
10647
|
rejectedCommandCount: Array.isArray(ret.rejectedCommands) ? ret.rejectedCommands.length : 0
|
|
9982
10648
|
}
|
|
9983
10649
|
node._assistantLog.push(assistantEntry)
|
|
9984
10650
|
while (node._assistantLog.length > 50) node._assistantLog.shift()
|
|
9985
|
-
rememberConversationTurn({ sessionId, question, reply: content })
|
|
10651
|
+
if (!deferCameraReply) rememberConversationTurn({ sessionId, question, reply: content })
|
|
9986
10652
|
const replyMessage = buildKnxAiReplyMessage({
|
|
9987
10653
|
inputMessage: msg,
|
|
9988
10654
|
content,
|
|
@@ -9996,6 +10662,7 @@ module.exports = function (RED) {
|
|
|
9996
10662
|
operationCount: preparedCommands.length,
|
|
9997
10663
|
commandCount: writeCommands.length,
|
|
9998
10664
|
readCount: readCommands.length,
|
|
10665
|
+
cameraActionCount: preparedCameraActions.length,
|
|
9999
10666
|
readResults: readResultMetadata,
|
|
10000
10667
|
awaitingConfirmation,
|
|
10001
10668
|
confirmationExpiresAt: confirmationRequest ? confirmationRequest.expiresAt : 0,
|
|
@@ -10005,7 +10672,10 @@ module.exports = function (RED) {
|
|
|
10005
10672
|
},
|
|
10006
10673
|
summary: emittedReadCommands.length > 0 ? rebuildCachedSummaryNow() : ret.summary
|
|
10007
10674
|
})
|
|
10008
|
-
if (
|
|
10675
|
+
if (deferCameraReply) {
|
|
10676
|
+
// The matching camera provider returns the snapshot asynchronously;
|
|
10677
|
+
// its image (and optional visual analysis) becomes the chat reply.
|
|
10678
|
+
} else if (emittedReadCommands.length > 0) {
|
|
10009
10679
|
if (!sendKnxAiOutputs([null, null, replyMessage, null], msg)) return
|
|
10010
10680
|
} else if (!sendKnxAiOutputs([null, null, replyMessage, commandMessages.length ? commandMessages : null], msg)) {
|
|
10011
10681
|
return
|
|
@@ -10199,6 +10869,10 @@ module.exports = function (RED) {
|
|
|
10199
10869
|
return
|
|
10200
10870
|
}
|
|
10201
10871
|
if (!adaptedMessage) return
|
|
10872
|
+
const adaptedTopic = String(adaptedMessage.topic || '').toLocaleLowerCase()
|
|
10873
|
+
if (adaptedTopic === 'ask' || adaptedTopic === 'chat' || adaptedTopic === 'question' || adaptedTopic === 'prompt') {
|
|
10874
|
+
rememberChatSessionSource({ sessionId: resolveKnxAiSessionId(adaptedMessage), msg })
|
|
10875
|
+
}
|
|
10202
10876
|
const p = handleCommand(adaptedMessage)
|
|
10203
10877
|
if (p && typeof p.catch === 'function') {
|
|
10204
10878
|
p.catch((error) => {
|
|
@@ -10219,6 +10893,14 @@ module.exports = function (RED) {
|
|
|
10219
10893
|
if (node._busConnectionWatchTimer) clearInterval(node._busConnectionWatchTimer)
|
|
10220
10894
|
if (node._homeMemoryPeriodicTimer) clearInterval(node._homeMemoryPeriodicTimer)
|
|
10221
10895
|
if (node._proactiveCheckTimer) clearInterval(node._proactiveCheckTimer)
|
|
10896
|
+
if (node._cameraRegistrySyncTimer) clearInterval(node._cameraRegistrySyncTimer)
|
|
10897
|
+
node._cameraRegistrySyncTimer = null
|
|
10898
|
+
try { if (typeof node._cameraRegistryUnsubscribe === 'function') node._cameraRegistryUnsubscribe() } catch (error) { /* ignore */ }
|
|
10899
|
+
node._cameraRegistryUnsubscribe = null
|
|
10900
|
+
node._cameraProviderUnsubscribers.forEach(unsubscribe => {
|
|
10901
|
+
try { if (typeof unsubscribe === 'function') unsubscribe() } catch (error) { /* ignore */ }
|
|
10902
|
+
})
|
|
10903
|
+
node._cameraProviderUnsubscribers.clear()
|
|
10222
10904
|
if (node._homeMemoryWriteTimer) {
|
|
10223
10905
|
clearTimeout(node._homeMemoryWriteTimer)
|
|
10224
10906
|
node._homeMemoryWriteTimer = null
|
|
@@ -10227,6 +10909,12 @@ module.exports = function (RED) {
|
|
|
10227
10909
|
clearTimeout(node._chatContextWriteTimer)
|
|
10228
10910
|
node._chatContextWriteTimer = null
|
|
10229
10911
|
}
|
|
10912
|
+
if (node._pendingCameraRequests instanceof Map) {
|
|
10913
|
+
node._pendingCameraRequests.forEach(pending => {
|
|
10914
|
+
try { if (pending && pending.timer) clearTimeout(pending.timer) } catch (error) { /* ignore */ }
|
|
10915
|
+
})
|
|
10916
|
+
node._pendingCameraRequests.clear()
|
|
10917
|
+
}
|
|
10230
10918
|
persistHomeMemoryNow()
|
|
10231
10919
|
persistChatContextNow()
|
|
10232
10920
|
if (node._homeMemoryStorePath) {
|
|
@@ -10284,6 +10972,25 @@ module.exports = function (RED) {
|
|
|
10284
10972
|
node.sysLogger?.warn(`KNX AI history startup error: ${error.message || error}`)
|
|
10285
10973
|
}
|
|
10286
10974
|
|
|
10975
|
+
try {
|
|
10976
|
+
const cameraRegistry = getKnxAiCameraAdapterRegistry()
|
|
10977
|
+
node._cameraRegistryUnsubscribe = cameraRegistry.subscribe(() => {
|
|
10978
|
+
Promise.resolve(syncCameraAdapterRegistry({ force: true })).catch(error => {
|
|
10979
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter refresh error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10980
|
+
})
|
|
10981
|
+
})
|
|
10982
|
+
Promise.resolve(syncCameraAdapterRegistry({ force: true })).catch(error => {
|
|
10983
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter startup error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10984
|
+
})
|
|
10985
|
+
node._cameraRegistrySyncTimer = setInterval(() => {
|
|
10986
|
+
Promise.resolve(syncCameraAdapterRegistry()).catch(error => {
|
|
10987
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter refresh error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10988
|
+
})
|
|
10989
|
+
}, 30 * 1000)
|
|
10990
|
+
} catch (error) {
|
|
10991
|
+
try { node.sysLogger?.warn(`KNX AI camera registry unavailable: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10992
|
+
}
|
|
10993
|
+
|
|
10287
10994
|
if (node._homeMemoryPeriodicTimer) clearInterval(node._homeMemoryPeriodicTimer)
|
|
10288
10995
|
node._homeMemoryPeriodicTimer = setInterval(() => {
|
|
10289
10996
|
try { persistHomeMemoryNow() } catch (error) { /* persistHomeMemoryNow already guards */ }
|
|
@@ -10314,6 +11021,7 @@ module.exports = function (RED) {
|
|
|
10314
11021
|
|
|
10315
11022
|
module.exports.__test = {
|
|
10316
11023
|
bindSharedKnxAiState,
|
|
11024
|
+
applyKnxAiChatMediaPresetFallback,
|
|
10317
11025
|
buildKnxAiPackageNodeCatalog,
|
|
10318
11026
|
buildKnxAiConfirmationRequest,
|
|
10319
11027
|
buildKnxAiUniversalMessage,
|
|
@@ -10339,5 +11047,6 @@ module.exports.__test = {
|
|
|
10339
11047
|
resolveKnxAiSessionId,
|
|
10340
11048
|
releaseSharedKnxAiState,
|
|
10341
11049
|
safeKnxAiSend,
|
|
11050
|
+
summarizeDetectedKnxAiCameraAdapters,
|
|
10342
11051
|
validateKnxAiPayloadForDpt
|
|
10343
11052
|
}
|