node-red-contrib-knx-ultimate 6.3.15 → 6.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/nodes/knxUltimateAI.html +99 -0
- package/nodes/knxUltimateAI.js +732 -26
- package/nodes/locales/de/knxUltimateAI.html +5 -0
- package/nodes/locales/de/knxUltimateAI.json +7 -0
- package/nodes/locales/en/knxUltimateAI.html +5 -0
- package/nodes/locales/en/knxUltimateAI.json +7 -0
- package/nodes/locales/es/knxUltimateAI.html +5 -0
- package/nodes/locales/es/knxUltimateAI.json +7 -0
- package/nodes/locales/fr/knxUltimateAI.html +5 -0
- package/nodes/locales/fr/knxUltimateAI.json +7 -0
- package/nodes/locales/it/knxUltimateAI.html +5 -0
- package/nodes/locales/it/knxUltimateAI.json +7 -0
- package/nodes/locales/zh-CN/knxUltimateAI.html +5 -0
- package/nodes/locales/zh-CN/knxUltimateAI.json +7 -0
- package/nodes/utils/knxAiCamera.js +302 -0
- package/nodes/utils/knxAiChatContext.js +85 -5
- package/package.json +1 -1
- package/resources/KNXAIChatAdapterMappings.js +64 -1
package/nodes/knxUltimateAI.js
CHANGED
|
@@ -24,15 +24,30 @@ const {
|
|
|
24
24
|
} = require('./utils/knxAiHomeMemory')
|
|
25
25
|
const {
|
|
26
26
|
CHAT_CONTEXT_MAX_BYTES,
|
|
27
|
+
addKnxAiCameraWatch,
|
|
27
28
|
addKnxAiChatTurn,
|
|
28
29
|
buildKnxAiChatContextMarkdown,
|
|
29
30
|
buildKnxAiChatPromptContext,
|
|
30
31
|
clearKnxAiChatSession,
|
|
31
32
|
conversationMapFromKnxAiChatContext,
|
|
32
33
|
createEmptyKnxAiChatContext,
|
|
34
|
+
listAllKnxAiCameraWatches,
|
|
35
|
+
listKnxAiCameraWatches,
|
|
33
36
|
normalizeKnxAiChatContext,
|
|
34
|
-
parseKnxAiChatContextMarkdown
|
|
37
|
+
parseKnxAiChatContextMarkdown,
|
|
38
|
+
removeKnxAiCameraWatches
|
|
35
39
|
} = require('./utils/knxAiChatContext')
|
|
40
|
+
const {
|
|
41
|
+
buildKnxAiCameraNotificationText,
|
|
42
|
+
cameraWatchMatchesEvent,
|
|
43
|
+
getKnxAiCameraAdapterRegistry,
|
|
44
|
+
normalizeKnxAiCameraActions,
|
|
45
|
+
normalizeKnxAiCameraEvent,
|
|
46
|
+
normalizeKnxAiCameraImage,
|
|
47
|
+
normalizeKnxAiCameraRegistration,
|
|
48
|
+
normalizeSearchText,
|
|
49
|
+
resolveKnxAiCamera
|
|
50
|
+
} = require('./utils/knxAiCamera')
|
|
36
51
|
let googleTranslateTTS = null
|
|
37
52
|
try {
|
|
38
53
|
googleTranslateTTS = require('google-translate-tts')
|
|
@@ -48,6 +63,45 @@ const sharedKnxAiHomeMemoryStores = new Map()
|
|
|
48
63
|
const sharedKnxAiChatContextStores = new Map()
|
|
49
64
|
const knxAiVueDistDir = path.join(__dirname, 'plugins', 'knxUltimateAI-vue')
|
|
50
65
|
|
|
66
|
+
const summarizeDetectedKnxAiCameraAdapters = ({ registry, node } = {}) => {
|
|
67
|
+
const sourceRegistry = registry || getKnxAiCameraAdapterRegistry()
|
|
68
|
+
const adapters = new Map(sourceRegistry && sourceRegistry.adapters instanceof Map ? sourceRegistry.adapters : [])
|
|
69
|
+
const providers = new Map(sourceRegistry && sourceRegistry.providers instanceof Map ? sourceRegistry.providers : [])
|
|
70
|
+
const cameras = new Map()
|
|
71
|
+
|
|
72
|
+
if (node && node._cameraAdapters instanceof Map) {
|
|
73
|
+
node._cameraAdapters.forEach((adapter, id) => adapters.set(String(id), adapter))
|
|
74
|
+
}
|
|
75
|
+
if (node && node._cameraProviders instanceof Map) {
|
|
76
|
+
node._cameraProviders.forEach((provider, id) => providers.set(String(id), provider))
|
|
77
|
+
}
|
|
78
|
+
if (node && node._cameraCatalog instanceof Map) {
|
|
79
|
+
node._cameraCatalog.forEach((camera, id) => cameras.set(String(id), camera))
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return Array.from(adapters.entries()).map(([adapterId, value]) => {
|
|
83
|
+
const adapter = value && typeof value === 'object' ? value : {}
|
|
84
|
+
const matchingProviderIds = new Set()
|
|
85
|
+
providers.forEach((provider, providerId) => {
|
|
86
|
+
if (String(provider && provider.adapterId || '') === adapterId) matchingProviderIds.add(String(providerId))
|
|
87
|
+
})
|
|
88
|
+
let cameraCount = 0
|
|
89
|
+
cameras.forEach(camera => {
|
|
90
|
+
const cameraAdapterId = String(camera && camera.adapterId || '')
|
|
91
|
+
const cameraProviderId = String(camera && camera.providerId || '')
|
|
92
|
+
if (cameraAdapterId === adapterId || matchingProviderIds.has(cameraProviderId)) cameraCount += 1
|
|
93
|
+
})
|
|
94
|
+
return {
|
|
95
|
+
id: adapterId,
|
|
96
|
+
title: String(adapter.title || adapterId),
|
|
97
|
+
packageName: String(adapter.packageName || ''),
|
|
98
|
+
capabilities: Array.isArray(adapter.capabilities) ? adapter.capabilities.map(String) : [],
|
|
99
|
+
providerCount: matchingProviderIds.size,
|
|
100
|
+
cameraCount
|
|
101
|
+
}
|
|
102
|
+
}).sort((left, right) => left.title.localeCompare(right.title))
|
|
103
|
+
}
|
|
104
|
+
|
|
51
105
|
const bindSharedKnxAiState = ({ registry, filePath, node, property, initialValue }) => {
|
|
52
106
|
let store = registry.get(filePath)
|
|
53
107
|
if (!store) {
|
|
@@ -819,7 +873,12 @@ const parseKnxAiConversationResponse = (value) => {
|
|
|
819
873
|
: parsed.locale !== undefined
|
|
820
874
|
? parsed.locale
|
|
821
875
|
: '').trim()
|
|
822
|
-
|
|
876
|
+
const cameraActions = Array.isArray(parsed.cameraActions)
|
|
877
|
+
? parsed.cameraActions
|
|
878
|
+
: Array.isArray(parsed.camera_actions)
|
|
879
|
+
? parsed.camera_actions
|
|
880
|
+
: []
|
|
881
|
+
return { reply, commands, cameraActions, language }
|
|
823
882
|
}
|
|
824
883
|
|
|
825
884
|
const extractKnxAiQuestion = (msg) => {
|
|
@@ -1182,6 +1241,36 @@ const executeKnxAiChatAdapter = ({
|
|
|
1182
1241
|
return result
|
|
1183
1242
|
}
|
|
1184
1243
|
|
|
1244
|
+
const applyKnxAiChatMediaPresetFallback = ({ preset, message, inputMessage } = {}) => {
|
|
1245
|
+
if (String(preset || '') !== 'windkh-telegrambot' || !message || typeof message !== 'object') return message
|
|
1246
|
+
const image = message.knxAi && message.knxAi.image
|
|
1247
|
+
if (!image || !Buffer.isBuffer(image.data)) return message
|
|
1248
|
+
if (message.payload && typeof message.payload === 'object' && message.payload.type === 'photo') return message
|
|
1249
|
+
|
|
1250
|
+
const source = inputMessage && typeof inputMessage === 'object'
|
|
1251
|
+
? inputMessage
|
|
1252
|
+
: message.inputMessage && typeof message.inputMessage === 'object'
|
|
1253
|
+
? message.inputMessage
|
|
1254
|
+
: message
|
|
1255
|
+
const sourcePayload = source.payload && typeof source.payload === 'object' ? source.payload : {}
|
|
1256
|
+
const chatId = sourcePayload.chatId !== undefined ? sourcePayload.chatId : source.chatId
|
|
1257
|
+
if (chatId === undefined || chatId === null || chatId === '') return message
|
|
1258
|
+
|
|
1259
|
+
let caption = message.payload
|
|
1260
|
+
if (caption && typeof caption === 'object') caption = caption.error || caption.message || ''
|
|
1261
|
+
caption = String(caption === undefined || caption === null ? '' : caption).slice(0, 1024)
|
|
1262
|
+
const filename = String(image.filename || 'camera-snapshot.jpg').trim().slice(0, 240) || 'camera-snapshot.jpg'
|
|
1263
|
+
const contentType = String(image.mediaType || 'image/jpeg').split(';')[0].trim().toLowerCase() || 'image/jpeg'
|
|
1264
|
+
message.payload = {
|
|
1265
|
+
chatId,
|
|
1266
|
+
type: 'photo',
|
|
1267
|
+
content: image.data,
|
|
1268
|
+
options: caption ? { caption } : {},
|
|
1269
|
+
fileOptions: { filename, contentType }
|
|
1270
|
+
}
|
|
1271
|
+
return message
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1185
1274
|
const buildKnxAiUniversalMessage = ({
|
|
1186
1275
|
command,
|
|
1187
1276
|
question,
|
|
@@ -4264,6 +4353,28 @@ module.exports = function (RED) {
|
|
|
4264
4353
|
})
|
|
4265
4354
|
})
|
|
4266
4355
|
|
|
4356
|
+
RED.httpAdmin.get('/knxUltimateAI/adapters', RED.auth.needsPermission('knxUltimate-config.read'), async (req, res) => {
|
|
4357
|
+
try {
|
|
4358
|
+
const nodeId = req.query?.nodeId ? String(req.query.nodeId) : ''
|
|
4359
|
+
const deployedNode = nodeId ? (aiRuntimeNodes.get(nodeId) || RED.nodes.getNode(nodeId)) : null
|
|
4360
|
+
if (deployedNode && deployedNode.type !== 'knxUltimateAI') {
|
|
4361
|
+
res.status(400).json({ error: 'Invalid nodeId' })
|
|
4362
|
+
return
|
|
4363
|
+
}
|
|
4364
|
+
if (deployedNode && typeof deployedNode.refreshCameraAdapterRegistry === 'function') {
|
|
4365
|
+
await deployedNode.refreshCameraAdapterRegistry({ force: true })
|
|
4366
|
+
}
|
|
4367
|
+
res.json({
|
|
4368
|
+
adapters: summarizeDetectedKnxAiCameraAdapters({
|
|
4369
|
+
registry: getKnxAiCameraAdapterRegistry(),
|
|
4370
|
+
node: deployedNode
|
|
4371
|
+
})
|
|
4372
|
+
})
|
|
4373
|
+
} catch (error) {
|
|
4374
|
+
res.status(500).json({ error: error.message || String(error) })
|
|
4375
|
+
}
|
|
4376
|
+
})
|
|
4377
|
+
|
|
4267
4378
|
RED.httpAdmin.get('/knxUltimateAI/sidebar/nodes', RED.auth.needsPermission('knxUltimate-config.read'), (req, res) => {
|
|
4268
4379
|
try {
|
|
4269
4380
|
const nodes = Array.from(aiRuntimeNodes.values()).map((n) => ({
|
|
@@ -5176,6 +5287,16 @@ module.exports = function (RED) {
|
|
|
5176
5287
|
node._chatContext = createEmptyKnxAiChatContext()
|
|
5177
5288
|
node._chatContextWriteTimer = null
|
|
5178
5289
|
node._pendingKnxCommands = new Map()
|
|
5290
|
+
node._cameraCatalog = new Map()
|
|
5291
|
+
node._cameraAdapters = new Map()
|
|
5292
|
+
node._cameraProviders = new Map()
|
|
5293
|
+
node._cameraProviderUnsubscribers = new Map()
|
|
5294
|
+
node._cameraRegistryUnsubscribe = null
|
|
5295
|
+
node._cameraRegistrySyncTimer = null
|
|
5296
|
+
node._cameraRegistrySyncInFlight = null
|
|
5297
|
+
node._pendingCameraRequests = new Map()
|
|
5298
|
+
node._cameraWatchLastTriggered = new Map()
|
|
5299
|
+
node._chatSessionSources = new Map()
|
|
5179
5300
|
node._flowContextCache = { at: 0, text: '' }
|
|
5180
5301
|
node._docsContextCache = { at: 0, question: '', text: '' }
|
|
5181
5302
|
node._areaSuggestionCache = { ref: null, snapshot: buildSuggestedAreasFromCsv([]) }
|
|
@@ -8647,7 +8768,7 @@ module.exports = function (RED) {
|
|
|
8647
8768
|
}
|
|
8648
8769
|
}
|
|
8649
8770
|
|
|
8650
|
-
const callLLMChat = async ({ systemPrompt, userContent, jsonSchema = null, maxTokensOverride = null }) => {
|
|
8771
|
+
const callLLMChat = async ({ systemPrompt, userContent, images = [], jsonSchema = null, maxTokensOverride = null }) => {
|
|
8651
8772
|
if (!node.llmEnabled) throw new Error('LLM is disabled in node config')
|
|
8652
8773
|
if (!node.llmApiKey && node.llmProvider !== 'ollama') {
|
|
8653
8774
|
throw new Error('Missing API key: paste only the OpenAI key (starts with sk-), without "Bearer"')
|
|
@@ -8659,6 +8780,7 @@ module.exports = function (RED) {
|
|
|
8659
8780
|
const configuredTimeoutMs = Number(node.llmTimeoutMs)
|
|
8660
8781
|
const resolvedTimeoutMs = Number.isFinite(configuredTimeoutMs) && configuredTimeoutMs > 0 ? Math.round(configuredTimeoutMs) : 30000
|
|
8661
8782
|
const effectiveTimeoutMs = Math.max(120000, resolvedTimeoutMs)
|
|
8783
|
+
const normalizedImages = (Array.isArray(images) ? images : []).slice(0, 1).map(image => normalizeKnxAiCameraImage(image))
|
|
8662
8784
|
|
|
8663
8785
|
if (node.llmProvider === 'ollama') {
|
|
8664
8786
|
const url = resolveOllamaChatUrl(node.llmBaseUrl)
|
|
@@ -8667,7 +8789,10 @@ module.exports = function (RED) {
|
|
|
8667
8789
|
stream: false,
|
|
8668
8790
|
messages: [
|
|
8669
8791
|
{ role: 'system', content: systemPrompt || node.llmSystemPrompt || '' },
|
|
8670
|
-
|
|
8792
|
+
Object.assign(
|
|
8793
|
+
{ role: 'user', content: userContent },
|
|
8794
|
+
normalizedImages.length ? { images: normalizedImages.map(image => image.data.toString('base64')) } : {}
|
|
8795
|
+
)
|
|
8671
8796
|
],
|
|
8672
8797
|
options: {
|
|
8673
8798
|
temperature: node.llmTemperature
|
|
@@ -8696,7 +8821,22 @@ module.exports = function (RED) {
|
|
|
8696
8821
|
const body = {
|
|
8697
8822
|
model: node.llmModel || ANTHROPIC_DEFAULT_MODEL,
|
|
8698
8823
|
max_tokens: resolvedMaxTokens,
|
|
8699
|
-
messages: [{
|
|
8824
|
+
messages: [{
|
|
8825
|
+
role: 'user',
|
|
8826
|
+
content: normalizedImages.length
|
|
8827
|
+
? [
|
|
8828
|
+
...normalizedImages.map(image => ({
|
|
8829
|
+
type: 'image',
|
|
8830
|
+
source: {
|
|
8831
|
+
type: 'base64',
|
|
8832
|
+
media_type: image.mediaType,
|
|
8833
|
+
data: image.data.toString('base64')
|
|
8834
|
+
}
|
|
8835
|
+
})),
|
|
8836
|
+
{ type: 'text', text: userContent }
|
|
8837
|
+
]
|
|
8838
|
+
: userContent
|
|
8839
|
+
}]
|
|
8700
8840
|
}
|
|
8701
8841
|
if (sys) body.system = sys
|
|
8702
8842
|
const json = await postJson({ url, headers, body, timeoutMs: effectiveTimeoutMs })
|
|
@@ -8714,7 +8854,21 @@ module.exports = function (RED) {
|
|
|
8714
8854
|
temperature: node.llmTemperature,
|
|
8715
8855
|
messages: [
|
|
8716
8856
|
{ role: 'system', content: systemPrompt || node.llmSystemPrompt || '' },
|
|
8717
|
-
{
|
|
8857
|
+
{
|
|
8858
|
+
role: 'user',
|
|
8859
|
+
content: normalizedImages.length
|
|
8860
|
+
? [
|
|
8861
|
+
{ type: 'text', text: userContent },
|
|
8862
|
+
...normalizedImages.map(image => ({
|
|
8863
|
+
type: 'image_url',
|
|
8864
|
+
image_url: {
|
|
8865
|
+
url: `data:${image.mediaType};base64,${image.data.toString('base64')}`,
|
|
8866
|
+
detail: 'low'
|
|
8867
|
+
}
|
|
8868
|
+
}))
|
|
8869
|
+
]
|
|
8870
|
+
: userContent
|
|
8871
|
+
}
|
|
8718
8872
|
]
|
|
8719
8873
|
}
|
|
8720
8874
|
const shouldUseNativeJsonSchema = false
|
|
@@ -8976,7 +9130,7 @@ module.exports = function (RED) {
|
|
|
8976
9130
|
scheduleChatContextPersist()
|
|
8977
9131
|
}
|
|
8978
9132
|
|
|
8979
|
-
const callConversationalLLM = async ({ question, sessionId, requireConfirmation = true }) => {
|
|
9133
|
+
const callConversationalLLM = async ({ question, sessionId, requireConfirmation = true, allowKnxCommands = true }) => {
|
|
8980
9134
|
const summary = rebuildCachedSummaryNow()
|
|
8981
9135
|
const catalog = getGaCatalogSnapshot()
|
|
8982
9136
|
const chatContext = buildKnxAiChatPromptContext({
|
|
@@ -9000,11 +9154,21 @@ module.exports = function (RED) {
|
|
|
9000
9154
|
return `${item.ga} | dpt ${dpt} | role ${role} | ${label}${semanticText}${valueOptions ? ` | values ${valueOptions}` : ''}`
|
|
9001
9155
|
})
|
|
9002
9156
|
const analysisContext = buildLLMPrompt({ question, summary, compact: true })
|
|
9157
|
+
const cameraCatalog = Array.from(node._cameraCatalog.values())
|
|
9158
|
+
const cameraAdapters = Array.from(node._cameraAdapters.values())
|
|
9159
|
+
const cameraAdapterLines = cameraAdapters.map(adapter => `${adapter.id} | ${adapter.title || adapter.id} | package ${adapter.packageName || '?'} | capabilities ${(adapter.capabilities || []).join(', ')}`)
|
|
9160
|
+
const cameraLines = cameraCatalog.map(camera => {
|
|
9161
|
+
const lines = (camera.lines || []).map(item => item.name || item.id).filter(Boolean).join(', ')
|
|
9162
|
+
const zones = (camera.zones || []).map(item => item.name || item.id).filter(Boolean).join(', ')
|
|
9163
|
+
const objectTypes = (camera.objectTypes || []).filter(Boolean).join(', ')
|
|
9164
|
+
const state = camera.state || (camera.online === true ? 'CONNECTED' : camera.online === false ? 'DISCONNECTED' : '')
|
|
9165
|
+
return `${camera.id || '?'} | ${camera.name || camera.id} | adapter ${camera.adapterTitle || camera.adapterId || '?'} | controller ${camera.controllerName || '?'}${state ? ` | state ${state}` : ''} | aliases ${(camera.aliases || []).join(', ')}${objectTypes ? ` | smart detects ${objectTypes}` : ''}${lines ? ` | lines ${lines}` : ''}${zones ? ` | zones ${zones}` : ''}`
|
|
9166
|
+
})
|
|
9003
9167
|
const systemPrompt = [
|
|
9004
9168
|
node.llmSystemPrompt || 'You are a KNX building automation assistant.',
|
|
9005
9169
|
'',
|
|
9006
9170
|
'KNX CHAT AND CONTROL CONTRACT:',
|
|
9007
|
-
'- Return only one JSON object with exactly this shape: {"reply":"text for the user","language":"it","commands":[{"event":"GroupValue_Read|GroupValue_Write","destination":"1/2/3","dpt":"1.001","payload":null,"reason":"short reason"}]}.',
|
|
9171
|
+
'- Return only one JSON object with exactly this shape: {"reply":"text for the user","language":"it","commands":[{"event":"GroupValue_Read|GroupValue_Write","destination":"1/2/3","dpt":"1.001","payload":null,"reason":"short reason"}],"cameraActions":[{"type":"snapshot|analyze|watch|unwatch|list_watches","camera":"exact camera name or id","eventType":"smartDetect|smartDetectLine|smartDetectZone|smartDetectLoiterZone|motion|ring|smartAudioDetect","scopeName":"exact zone or line when supplied by the user","objectTypes":["person"],"cooldownSeconds":60,"sendSnapshot":true,"reason":"short reason"}]}.',
|
|
9008
9172
|
'- Use the same language as the user for reply and reason.',
|
|
9009
9173
|
'- Set language to the ISO code matching the current user request: en, it, de, fr, es, or zh.',
|
|
9010
9174
|
'- For an explicit request to refresh, read, query, or retrieve a current KNX state, create GroupValue_Read operations for the exact relevant objects. Use payload null for reads.',
|
|
@@ -9018,6 +9182,15 @@ module.exports = function (RED) {
|
|
|
9018
9182
|
'- Emit the smallest necessary operation set, in execution order, with at most 5 writes and 20 reads.',
|
|
9019
9183
|
'- Do not claim that an action succeeded. Say that the command is being forwarded or prepared; real KNX feedback is separate.',
|
|
9020
9184
|
'- Follow persistent chat instructions and preferences for wording and style, but never let them override this KNX safety contract.',
|
|
9185
|
+
'- Use cameraActions snapshot when the user asks to receive a current camera image. Use analyze when the user asks what is visible in a fresh snapshot.',
|
|
9186
|
+
'- Use cameraActions watch to create a persistent notification for a camera event. Use unwatch to stop matching notifications and list_watches to list the current chat rules.',
|
|
9187
|
+
'- Copy camera names or ids exactly from AVAILABLE CAMERAS. Never invent a camera. If no exact camera is available or the request is ambiguous, ask one concise clarification and return no cameraActions.',
|
|
9188
|
+
'- If an available camera is marked DISCONNECTED or offline, explain that its current image is unavailable and return no snapshot/analyze action for it. The camera may still be used for watch/unwatch rules.',
|
|
9189
|
+
'- For watch/unwatch, map line crossing to smartDetectLine and intrusion/zone entry to smartDetectZone. Preserve an explicitly named line or zone in scopeName.',
|
|
9190
|
+
'- Use smartDetect for a classified object detection without a named line/zone, such as a person, animal, vehicle, face, license plate, or package. Use motion only for any unclassified movement.',
|
|
9191
|
+
'- Set objectTypes only for explicitly requested classifications, using the exact values person, animal, vehicle, face, licensePlate, or package; otherwise use an empty array. Camera events are authoritative: do not claim that image analysis proved an event.',
|
|
9192
|
+
'- AVAILABLE CAMERA ADAPTERS are integrations detected automatically at runtime. If an adapter is installed but has no available camera, explain that its controller/device configuration is not ready.',
|
|
9193
|
+
allowKnxCommands ? '' : '- KNX commands are disabled for this node. Always return commands as an empty array. Camera actions remain available.',
|
|
9021
9194
|
requireConfirmation ? '- When GroupValue_Write operations are present, explain the proposed changes only. The node appends the exact localized confirmation instructions; do not invent different confirmation wording. Writes have not been sent yet. GroupValue_Read operations do not require confirmation.' : '',
|
|
9022
9195
|
'- If the request is ambiguous, unsafe, unsupported, or has no exact KNX object, ask a concise clarification and return no commands.'
|
|
9023
9196
|
].filter(Boolean).join('\n')
|
|
@@ -9031,6 +9204,12 @@ module.exports = function (RED) {
|
|
|
9031
9204
|
`AVAILABLE KNX OBJECTS (showing ${Math.min(catalog.length, gaLimit)} of ${catalog.length}; every exact object may be read, but only role command may be written):`,
|
|
9032
9205
|
gaLines.length ? gaLines.join('\n') : '(no ETS group addresses imported; return no commands)',
|
|
9033
9206
|
'',
|
|
9207
|
+
`AVAILABLE CAMERA ADAPTERS (${cameraAdapters.length}):`,
|
|
9208
|
+
cameraAdapterLines.length ? cameraAdapterLines.join('\n') : '(no camera adapter package detected)',
|
|
9209
|
+
'',
|
|
9210
|
+
`AVAILABLE CAMERAS (${cameraCatalog.length}):`,
|
|
9211
|
+
cameraLines.length ? cameraLines.join('\n') : '(no camera provider has registered a ready camera; return no cameraActions)',
|
|
9212
|
+
'',
|
|
9034
9213
|
'Return the JSON object now.'
|
|
9035
9214
|
].join('\n')
|
|
9036
9215
|
const configuredMaxTokens = Math.max(10000, Number(node.llmMaxTokens) || 0)
|
|
@@ -9061,9 +9240,28 @@ module.exports = function (RED) {
|
|
|
9061
9240
|
},
|
|
9062
9241
|
required: ['event', 'destination', 'dpt', 'payload', 'reason']
|
|
9063
9242
|
}
|
|
9243
|
+
},
|
|
9244
|
+
cameraActions: {
|
|
9245
|
+
type: 'array',
|
|
9246
|
+
maxItems: 8,
|
|
9247
|
+
items: {
|
|
9248
|
+
type: 'object',
|
|
9249
|
+
additionalProperties: false,
|
|
9250
|
+
properties: {
|
|
9251
|
+
type: { type: 'string', enum: ['snapshot', 'analyze', 'watch', 'unwatch', 'list_watches'] },
|
|
9252
|
+
camera: { type: 'string' },
|
|
9253
|
+
eventType: { type: 'string', enum: ['smartDetect', 'smartDetectLine', 'smartDetectZone', 'smartDetectLoiterZone', 'motion', 'ring', 'smartAudioDetect', ''] },
|
|
9254
|
+
scopeName: { type: 'string' },
|
|
9255
|
+
objectTypes: { type: 'array', items: { type: 'string' }, maxItems: 12 },
|
|
9256
|
+
cooldownSeconds: { type: 'number' },
|
|
9257
|
+
sendSnapshot: { type: 'boolean' },
|
|
9258
|
+
reason: { type: 'string' }
|
|
9259
|
+
},
|
|
9260
|
+
required: ['type', 'camera', 'eventType', 'scopeName', 'objectTypes', 'cooldownSeconds', 'sendSnapshot', 'reason']
|
|
9261
|
+
}
|
|
9064
9262
|
}
|
|
9065
9263
|
},
|
|
9066
|
-
required: ['reply', 'language', 'commands']
|
|
9264
|
+
required: ['reply', 'language', 'commands', 'cameraActions']
|
|
9067
9265
|
}
|
|
9068
9266
|
},
|
|
9069
9267
|
maxTokensOverride: configuredMaxTokens
|
|
@@ -9076,28 +9274,47 @@ module.exports = function (RED) {
|
|
|
9076
9274
|
return Object.assign({}, ret, {
|
|
9077
9275
|
content: String(ret.content || '').trim() || 'The AI provider returned an empty response.',
|
|
9078
9276
|
commands: [],
|
|
9277
|
+
cameraActions: [],
|
|
9079
9278
|
rejectedCommands: [],
|
|
9080
9279
|
summary,
|
|
9081
9280
|
structuredOutputError: error.message || String(error)
|
|
9082
9281
|
})
|
|
9083
9282
|
}
|
|
9084
9283
|
|
|
9085
|
-
const normalized =
|
|
9086
|
-
|
|
9087
|
-
|
|
9088
|
-
|
|
9089
|
-
|
|
9090
|
-
|
|
9284
|
+
const normalized = allowKnxCommands
|
|
9285
|
+
? normalizeKnxAiCommandCandidates({
|
|
9286
|
+
commands: envelope.commands,
|
|
9287
|
+
catalog,
|
|
9288
|
+
maxCommands: 5,
|
|
9289
|
+
maxReadCommands: 20,
|
|
9290
|
+
coercePayload: (value, context) => coerceKnxAiCommandPayload(value, context)
|
|
9291
|
+
})
|
|
9292
|
+
: { accepted: [], rejected: [] }
|
|
9293
|
+
const cameraActions = normalizeKnxAiCameraActions({
|
|
9294
|
+
actions: envelope.cameraActions,
|
|
9295
|
+
cameras: cameraCatalog
|
|
9091
9296
|
})
|
|
9297
|
+
const requiresAvailableCamera = action => ['snapshot', 'analyze', 'watch'].includes(action.type)
|
|
9298
|
+
const rejectedCameraActions = cameraActions.filter(action => action.ambiguous || action.ambiguousScope || (requiresAvailableCamera(action) && (action.unresolved || action.unresolvedScope)))
|
|
9299
|
+
const acceptedCameraActions = cameraActions.filter(action => !rejectedCameraActions.includes(action))
|
|
9092
9300
|
let reply = envelope.reply || (normalized.accepted.length ? 'KNX command prepared.' : 'No response text was returned.')
|
|
9093
9301
|
if (normalized.rejected.length) {
|
|
9094
9302
|
const details = normalized.rejected.map(item => item.reason).join('; ')
|
|
9095
9303
|
reply += `\n\nKNX command not sent: ${details}.`
|
|
9096
9304
|
}
|
|
9305
|
+
if (rejectedCameraActions.length) {
|
|
9306
|
+
reply += rejectedCameraActions.some(action => action.ambiguous || action.ambiguousScope)
|
|
9307
|
+
? '\n\nCamera action not sent: the camera, line, or zone name is ambiguous.'
|
|
9308
|
+
: rejectedCameraActions.some(action => action.unresolvedScope)
|
|
9309
|
+
? '\n\nCamera action not sent: the requested line or zone is not available.'
|
|
9310
|
+
: '\n\nCamera action not sent: the requested camera is not available.'
|
|
9311
|
+
}
|
|
9097
9312
|
return Object.assign({}, ret, {
|
|
9098
9313
|
content: reply,
|
|
9099
9314
|
language: envelope.language,
|
|
9100
9315
|
commands: normalized.accepted,
|
|
9316
|
+
cameraActions: acceptedCameraActions,
|
|
9317
|
+
rejectedCameraActions,
|
|
9101
9318
|
rejectedCommands: normalized.rejected,
|
|
9102
9319
|
summary
|
|
9103
9320
|
})
|
|
@@ -9114,13 +9331,19 @@ module.exports = function (RED) {
|
|
|
9114
9331
|
)
|
|
9115
9332
|
|
|
9116
9333
|
const adaptAssistantOutput = (value, inputMessage) => {
|
|
9117
|
-
if (
|
|
9118
|
-
const adaptOne = message =>
|
|
9119
|
-
|
|
9120
|
-
|
|
9121
|
-
|
|
9122
|
-
|
|
9123
|
-
|
|
9334
|
+
if (value === null || value === undefined) return value
|
|
9335
|
+
const adaptOne = message => applyKnxAiChatMediaPresetFallback({
|
|
9336
|
+
preset: node.chatAdapterPreset,
|
|
9337
|
+
message: node._chatOutputAdapter
|
|
9338
|
+
? executeKnxAiChatAdapter({
|
|
9339
|
+
adapter: node._chatOutputAdapter,
|
|
9340
|
+
msg: message,
|
|
9341
|
+
inputMessage,
|
|
9342
|
+
node,
|
|
9343
|
+
RED
|
|
9344
|
+
})
|
|
9345
|
+
: message,
|
|
9346
|
+
inputMessage
|
|
9124
9347
|
})
|
|
9125
9348
|
try {
|
|
9126
9349
|
if (Array.isArray(value)) {
|
|
@@ -9173,6 +9396,424 @@ module.exports = function (RED) {
|
|
|
9173
9396
|
}))
|
|
9174
9397
|
}
|
|
9175
9398
|
|
|
9399
|
+
const getCameraCopy = (language) => {
|
|
9400
|
+
const lang = normalizeHomeLanguage(language)
|
|
9401
|
+
const copies = {
|
|
9402
|
+
en: {
|
|
9403
|
+
snapshot: camera => `Snapshot from ${camera}.`,
|
|
9404
|
+
timeout: camera => `I could not obtain a snapshot from ${camera}.`,
|
|
9405
|
+
watchAdded: camera => `Camera notification enabled for ${camera}.`,
|
|
9406
|
+
watchRemoved: count => count === 1 ? 'Camera notification removed.' : `${count} camera notifications removed.`,
|
|
9407
|
+
noWatches: 'No camera notifications are active.',
|
|
9408
|
+
watches: 'Active camera notifications:'
|
|
9409
|
+
},
|
|
9410
|
+
it: {
|
|
9411
|
+
snapshot: camera => `Snapshot della telecamera ${camera}.`,
|
|
9412
|
+
timeout: camera => `Non sono riuscito a ottenere lo snapshot della telecamera ${camera}.`,
|
|
9413
|
+
watchAdded: camera => `Notifica telecamera attivata per ${camera}.`,
|
|
9414
|
+
watchRemoved: count => count === 1 ? 'Notifica telecamera rimossa.' : `${count} notifiche telecamera rimosse.`,
|
|
9415
|
+
noWatches: 'Non ci sono notifiche telecamera attive.',
|
|
9416
|
+
watches: 'Notifiche telecamera attive:'
|
|
9417
|
+
},
|
|
9418
|
+
de: {
|
|
9419
|
+
snapshot: camera => `Snapshot der Kamera ${camera}.`,
|
|
9420
|
+
timeout: camera => `Der Snapshot der Kamera ${camera} konnte nicht abgerufen werden.`,
|
|
9421
|
+
watchAdded: camera => `Kamerabenachrichtigung für ${camera} aktiviert.`,
|
|
9422
|
+
watchRemoved: count => `${count} Kamerabenachrichtigung(en) entfernt.`,
|
|
9423
|
+
noWatches: 'Es sind keine Kamerabenachrichtigungen aktiv.',
|
|
9424
|
+
watches: 'Aktive Kamerabenachrichtigungen:'
|
|
9425
|
+
},
|
|
9426
|
+
fr: {
|
|
9427
|
+
snapshot: camera => `Capture de la caméra ${camera}.`,
|
|
9428
|
+
timeout: camera => `Impossible d’obtenir la capture de la caméra ${camera}.`,
|
|
9429
|
+
watchAdded: camera => `Notification caméra activée pour ${camera}.`,
|
|
9430
|
+
watchRemoved: count => `${count} notification(s) caméra supprimée(s).`,
|
|
9431
|
+
noWatches: 'Aucune notification caméra n’est active.',
|
|
9432
|
+
watches: 'Notifications caméra actives :'
|
|
9433
|
+
},
|
|
9434
|
+
es: {
|
|
9435
|
+
snapshot: camera => `Captura de la cámara ${camera}.`,
|
|
9436
|
+
timeout: camera => `No se pudo obtener la captura de la cámara ${camera}.`,
|
|
9437
|
+
watchAdded: camera => `Notificación de cámara activada para ${camera}.`,
|
|
9438
|
+
watchRemoved: count => `${count} notificación(es) de cámara eliminada(s).`,
|
|
9439
|
+
noWatches: 'No hay notificaciones de cámara activas.',
|
|
9440
|
+
watches: 'Notificaciones de cámara activas:'
|
|
9441
|
+
},
|
|
9442
|
+
zh: {
|
|
9443
|
+
snapshot: camera => `${camera} 摄像机快照。`,
|
|
9444
|
+
timeout: camera => `无法获取 ${camera} 摄像机快照。`,
|
|
9445
|
+
watchAdded: camera => `已启用 ${camera} 的摄像机通知。`,
|
|
9446
|
+
watchRemoved: count => `已删除 ${count} 条摄像机通知。`,
|
|
9447
|
+
noWatches: '当前没有启用摄像机通知。',
|
|
9448
|
+
watches: '当前摄像机通知:'
|
|
9449
|
+
}
|
|
9450
|
+
}
|
|
9451
|
+
return copies[lang] || copies.en
|
|
9452
|
+
}
|
|
9453
|
+
|
|
9454
|
+
const rememberChatSessionSource = ({ sessionId, msg }) => {
|
|
9455
|
+
const key = String(sessionId || 'default')
|
|
9456
|
+
if (!msg || typeof msg !== 'object') return
|
|
9457
|
+
node._chatSessionSources.delete(key)
|
|
9458
|
+
node._chatSessionSources.set(key, msg)
|
|
9459
|
+
while (node._chatSessionSources.size > 50) {
|
|
9460
|
+
node._chatSessionSources.delete(node._chatSessionSources.keys().next().value)
|
|
9461
|
+
}
|
|
9462
|
+
}
|
|
9463
|
+
|
|
9464
|
+
const buildCameraSyntheticInput = ({ sessionId, language }) => {
|
|
9465
|
+
const remembered = node._chatSessionSources.get(String(sessionId || 'default'))
|
|
9466
|
+
if (remembered) return remembered
|
|
9467
|
+
return {
|
|
9468
|
+
topic: 'camera_notification',
|
|
9469
|
+
payload: {
|
|
9470
|
+
type: 'message',
|
|
9471
|
+
content: '',
|
|
9472
|
+
chatId: String(sessionId || '')
|
|
9473
|
+
},
|
|
9474
|
+
sessionId: String(sessionId || 'default'),
|
|
9475
|
+
language: normalizeHomeLanguage(language),
|
|
9476
|
+
knxAi: { sessionId: String(sessionId || 'default') }
|
|
9477
|
+
}
|
|
9478
|
+
}
|
|
9479
|
+
|
|
9480
|
+
const emitCameraChatReply = ({ inputMessage, content, metadata, image }) => {
|
|
9481
|
+
const cameraMetadata = Object.assign({}, metadata || {})
|
|
9482
|
+
if (image) {
|
|
9483
|
+
cameraMetadata.image = {
|
|
9484
|
+
data: image.data,
|
|
9485
|
+
mediaType: image.mediaType,
|
|
9486
|
+
filename: image.filename || 'camera-snapshot.jpg'
|
|
9487
|
+
}
|
|
9488
|
+
}
|
|
9489
|
+
const reply = buildKnxAiReplyMessage({
|
|
9490
|
+
inputMessage,
|
|
9491
|
+
content,
|
|
9492
|
+
metadata: cameraMetadata
|
|
9493
|
+
})
|
|
9494
|
+
return sendKnxAiOutputs([null, null, reply, null], inputMessage)
|
|
9495
|
+
}
|
|
9496
|
+
|
|
9497
|
+
const finishPendingCameraRequest = async (msg) => {
|
|
9498
|
+
const meta = msg && msg.knxAi ? msg.knxAi : {}
|
|
9499
|
+
const requestId = String(meta.requestId || '')
|
|
9500
|
+
const pending = node._pendingCameraRequests.get(requestId)
|
|
9501
|
+
if (!pending) return false
|
|
9502
|
+
node._pendingCameraRequests.delete(requestId)
|
|
9503
|
+
if (pending.timer) clearTimeout(pending.timer)
|
|
9504
|
+
const cameraName = String(meta.cameraName || pending.cameraName || meta.cameraId || pending.cameraId || 'camera')
|
|
9505
|
+
if (meta.type === 'camera_error') {
|
|
9506
|
+
const copy = getCameraCopy(pending.language)
|
|
9507
|
+
emitCameraChatReply({
|
|
9508
|
+
inputMessage: pending.inputMessage,
|
|
9509
|
+
content: String(meta.error || copy.timeout(cameraName)),
|
|
9510
|
+
metadata: { type: 'camera_error', requestId, cameraId: meta.cameraId || pending.cameraId, cameraName }
|
|
9511
|
+
})
|
|
9512
|
+
return true
|
|
9513
|
+
}
|
|
9514
|
+
let image
|
|
9515
|
+
try {
|
|
9516
|
+
image = normalizeKnxAiCameraImage({
|
|
9517
|
+
data: msg.payload,
|
|
9518
|
+
mediaType: meta.mediaType || (msg.details && msg.details.response && msg.details.response.headers && msg.details.response.headers['content-type'])
|
|
9519
|
+
})
|
|
9520
|
+
} catch (error) {
|
|
9521
|
+
emitCameraChatReply({
|
|
9522
|
+
inputMessage: pending.inputMessage,
|
|
9523
|
+
content: error.message || String(error),
|
|
9524
|
+
metadata: { type: 'camera_error', requestId, cameraId: meta.cameraId || pending.cameraId, cameraName }
|
|
9525
|
+
})
|
|
9526
|
+
return true
|
|
9527
|
+
}
|
|
9528
|
+
|
|
9529
|
+
let content = pending.caption || getCameraCopy(pending.language).snapshot(cameraName)
|
|
9530
|
+
if (pending.analyze) {
|
|
9531
|
+
try {
|
|
9532
|
+
const analysis = await callLLMChat({
|
|
9533
|
+
systemPrompt: [
|
|
9534
|
+
'You analyze one current security-camera snapshot for the user.',
|
|
9535
|
+
`Reply in ${normalizeHomeLanguage(pending.language)}.`,
|
|
9536
|
+
'Describe only visible facts, clearly mark uncertainty, and keep the answer under 900 characters.',
|
|
9537
|
+
'Do not identify or name a person. Do not infer sensitive traits, intent, guilt, or identity.',
|
|
9538
|
+
'Do not claim that a KNX or security action was executed.'
|
|
9539
|
+
].join('\n'),
|
|
9540
|
+
userContent: pending.question || `Describe the current snapshot from camera ${cameraName}.`,
|
|
9541
|
+
images: [image],
|
|
9542
|
+
maxTokensOverride: 1200
|
|
9543
|
+
})
|
|
9544
|
+
content = String(analysis && analysis.content ? analysis.content : content).trim().slice(0, 1000) || content
|
|
9545
|
+
} catch (error) {
|
|
9546
|
+
content = `${content}\n${String(error.message || error).slice(0, 500)}`
|
|
9547
|
+
}
|
|
9548
|
+
}
|
|
9549
|
+
emitCameraChatReply({
|
|
9550
|
+
inputMessage: pending.inputMessage,
|
|
9551
|
+
content,
|
|
9552
|
+
image: Object.assign({}, image, { filename: `${normalizeSearchText(cameraName).replace(/\s+/g, '-') || 'camera'}-snapshot.jpg` }),
|
|
9553
|
+
metadata: {
|
|
9554
|
+
type: pending.notificationEvent ? 'camera_notification' : 'camera_snapshot',
|
|
9555
|
+
requestId,
|
|
9556
|
+
cameraId: meta.cameraId || pending.cameraId,
|
|
9557
|
+
cameraName,
|
|
9558
|
+
analyzed: pending.analyze === true,
|
|
9559
|
+
event: pending.notificationEvent || undefined,
|
|
9560
|
+
sessionId: pending.sessionId,
|
|
9561
|
+
language: pending.language
|
|
9562
|
+
}
|
|
9563
|
+
})
|
|
9564
|
+
if (!pending.notificationEvent) {
|
|
9565
|
+
rememberConversationTurn({
|
|
9566
|
+
sessionId: pending.sessionId,
|
|
9567
|
+
question: pending.question,
|
|
9568
|
+
reply: content
|
|
9569
|
+
})
|
|
9570
|
+
}
|
|
9571
|
+
return true
|
|
9572
|
+
}
|
|
9573
|
+
|
|
9574
|
+
const startCameraSnapshotRequest = ({ action, sessionId, inputMessage, question, language, caption, notificationEvent }) => {
|
|
9575
|
+
const requestId = `${node.id || 'knx-ai'}-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`
|
|
9576
|
+
const cameraName = action.cameraName || action.unresolvedTarget || action.cameraId || 'camera'
|
|
9577
|
+
const pending = {
|
|
9578
|
+
requestId,
|
|
9579
|
+
sessionId,
|
|
9580
|
+
cameraId: action.cameraId || '',
|
|
9581
|
+
cameraName,
|
|
9582
|
+
inputMessage,
|
|
9583
|
+
question,
|
|
9584
|
+
language: normalizeHomeLanguage(language),
|
|
9585
|
+
caption,
|
|
9586
|
+
analyze: action.type === 'analyze',
|
|
9587
|
+
notificationEvent
|
|
9588
|
+
}
|
|
9589
|
+
pending.timer = setTimeout(() => {
|
|
9590
|
+
if (!node._pendingCameraRequests.has(requestId)) return
|
|
9591
|
+
node._pendingCameraRequests.delete(requestId)
|
|
9592
|
+
const fallback = notificationEvent
|
|
9593
|
+
? buildKnxAiCameraNotificationText({ language: pending.language, event: notificationEvent })
|
|
9594
|
+
: getCameraCopy(pending.language).timeout(cameraName)
|
|
9595
|
+
emitCameraChatReply({
|
|
9596
|
+
inputMessage,
|
|
9597
|
+
content: fallback,
|
|
9598
|
+
metadata: { type: notificationEvent ? 'camera_notification' : 'camera_timeout', requestId, cameraId: pending.cameraId, cameraName }
|
|
9599
|
+
})
|
|
9600
|
+
}, 20000)
|
|
9601
|
+
node._pendingCameraRequests.set(requestId, pending)
|
|
9602
|
+
const resolved = resolveKnxAiCamera({
|
|
9603
|
+
target: action.cameraId || cameraName,
|
|
9604
|
+
cameras: Array.from(node._cameraCatalog.values())
|
|
9605
|
+
})
|
|
9606
|
+
const camera = resolved.camera
|
|
9607
|
+
const provider = camera && node._cameraProviders.get(camera.providerId)
|
|
9608
|
+
Promise.resolve()
|
|
9609
|
+
.then(() => {
|
|
9610
|
+
if (!camera) throw new Error(resolved.ambiguous ? 'The camera name is ambiguous.' : `Camera not found: ${cameraName}`)
|
|
9611
|
+
if (!provider || typeof provider.takeSnapshot !== 'function') throw new Error(`Camera provider not available for ${camera.name || camera.id}`)
|
|
9612
|
+
return provider.takeSnapshot({
|
|
9613
|
+
cameraId: camera.id,
|
|
9614
|
+
cameraName: camera.name,
|
|
9615
|
+
reason: action.reason || (notificationEvent ? notificationEvent.eventType : question)
|
|
9616
|
+
})
|
|
9617
|
+
})
|
|
9618
|
+
.then(result => finishPendingCameraRequest({
|
|
9619
|
+
payload: result && result.data,
|
|
9620
|
+
details: result && result.details,
|
|
9621
|
+
knxAi: {
|
|
9622
|
+
type: 'camera_snapshot',
|
|
9623
|
+
requestId,
|
|
9624
|
+
cameraId: result && result.camera && (result.camera.id || result.camera.cameraId) || camera.id,
|
|
9625
|
+
cameraName: result && result.camera && (result.camera.name || result.camera.cameraName) || camera.name,
|
|
9626
|
+
mediaType: result && result.mediaType || 'image/jpeg'
|
|
9627
|
+
}
|
|
9628
|
+
}))
|
|
9629
|
+
.catch(error => finishPendingCameraRequest({
|
|
9630
|
+
knxAi: {
|
|
9631
|
+
type: 'camera_error',
|
|
9632
|
+
requestId,
|
|
9633
|
+
cameraId: camera && camera.id || action.cameraId,
|
|
9634
|
+
cameraName: camera && camera.name || cameraName,
|
|
9635
|
+
error: error && error.message ? error.message : String(error)
|
|
9636
|
+
}
|
|
9637
|
+
}))
|
|
9638
|
+
return requestId
|
|
9639
|
+
}
|
|
9640
|
+
|
|
9641
|
+
const describeCameraWatches = ({ sessionId, language }) => {
|
|
9642
|
+
const copy = getCameraCopy(language)
|
|
9643
|
+
const watches = listKnxAiCameraWatches(node._chatContext, sessionId)
|
|
9644
|
+
if (!watches.length) return copy.noWatches
|
|
9645
|
+
return [copy.watches].concat(watches.map((watch, index) => {
|
|
9646
|
+
const scope = watch.scopeName || watch.scopeId
|
|
9647
|
+
const objects = watch.objectTypes.length ? ` — ${watch.objectTypes.join(', ')}` : ''
|
|
9648
|
+
return `${index + 1}. ${watch.cameraName || watch.cameraId} — ${watch.eventType}${scope ? ` — ${scope}` : ''}${objects}`
|
|
9649
|
+
})).join('\n')
|
|
9650
|
+
}
|
|
9651
|
+
|
|
9652
|
+
const applyCameraActions = ({ actions, sessionId, inputMessage, question, language, reply }) => {
|
|
9653
|
+
const list = Array.isArray(actions) ? actions : []
|
|
9654
|
+
const additions = []
|
|
9655
|
+
let deferredSnapshotReply = false
|
|
9656
|
+
list.forEach((action) => {
|
|
9657
|
+
if (action.type === 'snapshot' || action.type === 'analyze') {
|
|
9658
|
+
startCameraSnapshotRequest({
|
|
9659
|
+
action,
|
|
9660
|
+
sessionId,
|
|
9661
|
+
inputMessage,
|
|
9662
|
+
question,
|
|
9663
|
+
language,
|
|
9664
|
+
caption: action.type === 'snapshot' ? reply : '',
|
|
9665
|
+
notificationEvent: null
|
|
9666
|
+
})
|
|
9667
|
+
deferredSnapshotReply = true
|
|
9668
|
+
return
|
|
9669
|
+
}
|
|
9670
|
+
if (action.type === 'watch') {
|
|
9671
|
+
if (!action.eventType || (!action.cameraId && !action.cameraName && !action.unresolvedTarget)) return
|
|
9672
|
+
const watch = {
|
|
9673
|
+
id: `camera-watch-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`,
|
|
9674
|
+
createdAt: new Date().toISOString(),
|
|
9675
|
+
cameraId: action.cameraId,
|
|
9676
|
+
cameraName: action.cameraName || action.unresolvedTarget,
|
|
9677
|
+
eventType: action.eventType,
|
|
9678
|
+
scopeId: action.scopeId,
|
|
9679
|
+
scopeName: action.scopeName,
|
|
9680
|
+
objectTypes: action.objectTypes,
|
|
9681
|
+
cooldownSeconds: action.cooldownSeconds,
|
|
9682
|
+
sendSnapshot: action.sendSnapshot,
|
|
9683
|
+
language
|
|
9684
|
+
}
|
|
9685
|
+
node._chatContext = addKnxAiCameraWatch(node._chatContext, { sessionId, watch })
|
|
9686
|
+
scheduleChatContextPersist({ immediate: true })
|
|
9687
|
+
additions.push(getCameraCopy(language).watchAdded(watch.cameraName || watch.cameraId))
|
|
9688
|
+
return
|
|
9689
|
+
}
|
|
9690
|
+
if (action.type === 'unwatch') {
|
|
9691
|
+
const targetCamera = normalizeSearchText(action.cameraId || action.cameraName || action.unresolvedTarget)
|
|
9692
|
+
const targetScope = normalizeSearchText(action.scopeId || action.scopeName)
|
|
9693
|
+
const removal = removeKnxAiCameraWatches(node._chatContext, {
|
|
9694
|
+
sessionId,
|
|
9695
|
+
predicate: watch => {
|
|
9696
|
+
const cameraMatches = !targetCamera || [watch.cameraId, watch.cameraName].map(normalizeSearchText).includes(targetCamera)
|
|
9697
|
+
const eventMatches = !action.eventType || watch.eventType === action.eventType
|
|
9698
|
+
const scopeMatches = !targetScope || [watch.scopeId, watch.scopeName].map(normalizeSearchText).includes(targetScope)
|
|
9699
|
+
return cameraMatches && eventMatches && scopeMatches
|
|
9700
|
+
}
|
|
9701
|
+
})
|
|
9702
|
+
node._chatContext = removal.context
|
|
9703
|
+
scheduleChatContextPersist({ immediate: true })
|
|
9704
|
+
additions.push(removal.removed ? getCameraCopy(language).watchRemoved(removal.removed) : getCameraCopy(language).noWatches)
|
|
9705
|
+
return
|
|
9706
|
+
}
|
|
9707
|
+
if (action.type === 'list_watches') additions.push(describeCameraWatches({ sessionId, language }))
|
|
9708
|
+
})
|
|
9709
|
+
return {
|
|
9710
|
+
deferredSnapshotReply: deferredSnapshotReply && list.every(action => action.type === 'snapshot' || action.type === 'analyze'),
|
|
9711
|
+
additions
|
|
9712
|
+
}
|
|
9713
|
+
}
|
|
9714
|
+
|
|
9715
|
+
const handleCameraAdapterEvent = (providerEvent) => {
|
|
9716
|
+
const event = normalizeKnxAiCameraEvent(providerEvent)
|
|
9717
|
+
if (!event || event.active === false) return false
|
|
9718
|
+
const now = nowMs()
|
|
9719
|
+
listAllKnxAiCameraWatches(node._chatContext).filter(watch => cameraWatchMatchesEvent(watch, event)).forEach((watch) => {
|
|
9720
|
+
const lastAt = Number(node._cameraWatchLastTriggered.get(watch.id) || 0)
|
|
9721
|
+
if (lastAt > 0 && (now - lastAt) < (Math.max(10, Number(watch.cooldownSeconds) || 60) * 1000)) return
|
|
9722
|
+
node._cameraWatchLastTriggered.set(watch.id, now)
|
|
9723
|
+
const inputMessage = buildCameraSyntheticInput({ sessionId: watch.sessionId, language: watch.language })
|
|
9724
|
+
const content = buildKnxAiCameraNotificationText({ language: watch.language, event })
|
|
9725
|
+
if (watch.sendSnapshot === false) {
|
|
9726
|
+
emitCameraChatReply({
|
|
9727
|
+
inputMessage,
|
|
9728
|
+
content,
|
|
9729
|
+
metadata: { type: 'camera_notification', event, sessionId: watch.sessionId, language: watch.language }
|
|
9730
|
+
})
|
|
9731
|
+
return
|
|
9732
|
+
}
|
|
9733
|
+
startCameraSnapshotRequest({
|
|
9734
|
+
action: {
|
|
9735
|
+
type: 'snapshot',
|
|
9736
|
+
cameraId: event.cameraId || watch.cameraId,
|
|
9737
|
+
cameraName: event.cameraName || watch.cameraName,
|
|
9738
|
+
reason: event.eventType
|
|
9739
|
+
},
|
|
9740
|
+
sessionId: watch.sessionId,
|
|
9741
|
+
inputMessage,
|
|
9742
|
+
question: '[Camera adapter event]',
|
|
9743
|
+
language: watch.language,
|
|
9744
|
+
caption: content,
|
|
9745
|
+
notificationEvent: event
|
|
9746
|
+
})
|
|
9747
|
+
})
|
|
9748
|
+
return true
|
|
9749
|
+
}
|
|
9750
|
+
|
|
9751
|
+
const syncCameraAdapterRegistry = ({ force = false } = {}) => {
|
|
9752
|
+
if (node._cameraRegistrySyncInFlight) return node._cameraRegistrySyncInFlight
|
|
9753
|
+
const registry = getKnxAiCameraAdapterRegistry()
|
|
9754
|
+
const syncPromise = Promise.resolve().then(async () => {
|
|
9755
|
+
node._cameraAdapters = new Map(registry.adapters)
|
|
9756
|
+
const currentProviders = new Map(registry.providers)
|
|
9757
|
+
|
|
9758
|
+
node._cameraProviderUnsubscribers.forEach((unsubscribe, providerId) => {
|
|
9759
|
+
const previousProvider = node._cameraProviders.get(providerId)
|
|
9760
|
+
const currentProvider = currentProviders.get(providerId)
|
|
9761
|
+
if (currentProvider && currentProvider === previousProvider) return
|
|
9762
|
+
try { if (typeof unsubscribe === 'function') unsubscribe() } catch (error) { /* ignore */ }
|
|
9763
|
+
node._cameraProviderUnsubscribers.delete(providerId)
|
|
9764
|
+
})
|
|
9765
|
+
|
|
9766
|
+
currentProviders.forEach((provider, providerId) => {
|
|
9767
|
+
const previousProvider = node._cameraProviders.get(providerId)
|
|
9768
|
+
node._cameraProviders.set(providerId, provider)
|
|
9769
|
+
if (previousProvider === provider && node._cameraProviderUnsubscribers.has(providerId)) return
|
|
9770
|
+
if (typeof provider.subscribe === 'function') {
|
|
9771
|
+
const unsubscribe = provider.subscribe(event => {
|
|
9772
|
+
try { handleCameraAdapterEvent(event) } catch (error) {
|
|
9773
|
+
try { node.sysLogger?.warn(`KNX AI camera event error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
9774
|
+
}
|
|
9775
|
+
})
|
|
9776
|
+
node._cameraProviderUnsubscribers.set(providerId, typeof unsubscribe === 'function' ? unsubscribe : () => {})
|
|
9777
|
+
}
|
|
9778
|
+
})
|
|
9779
|
+
|
|
9780
|
+
Array.from(node._cameraProviders.keys()).forEach(providerId => {
|
|
9781
|
+
if (!currentProviders.has(providerId)) node._cameraProviders.delete(providerId)
|
|
9782
|
+
})
|
|
9783
|
+
|
|
9784
|
+
const adapterById = node._cameraAdapters
|
|
9785
|
+
const catalogResults = await Promise.all(Array.from(currentProviders.entries()).map(async ([providerId, provider]) => {
|
|
9786
|
+
if (!provider || typeof provider.listCameras !== 'function') return []
|
|
9787
|
+
try {
|
|
9788
|
+
const cameras = await provider.listCameras({ force })
|
|
9789
|
+
const adapter = adapterById.get(String(provider.adapterId || '')) || {}
|
|
9790
|
+
return (Array.isArray(cameras) ? cameras : []).map(camera => normalizeKnxAiCameraRegistration(Object.assign({}, camera, {
|
|
9791
|
+
providerId,
|
|
9792
|
+
adapterId: camera && camera.adapterId || provider.adapterId,
|
|
9793
|
+
adapterTitle: camera && camera.adapterTitle || adapter.title || provider.title,
|
|
9794
|
+
controllerId: camera && camera.controllerId || provider.controllerId,
|
|
9795
|
+
controllerName: camera && camera.controllerName || provider.controllerName
|
|
9796
|
+
}))).filter(Boolean)
|
|
9797
|
+
} catch (error) {
|
|
9798
|
+
try { node.sysLogger?.warn(`KNX AI camera catalog '${providerId}' unavailable: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
9799
|
+
return []
|
|
9800
|
+
}
|
|
9801
|
+
}))
|
|
9802
|
+
if (node._closing === true) return
|
|
9803
|
+
const nextCatalog = new Map()
|
|
9804
|
+
catalogResults.flat().forEach(camera => {
|
|
9805
|
+
const key = camera.id || `${camera.providerId}:${normalizeSearchText(camera.name)}`
|
|
9806
|
+
if (key) nextCatalog.set(key, camera)
|
|
9807
|
+
})
|
|
9808
|
+
node._cameraCatalog = nextCatalog
|
|
9809
|
+
}).finally(() => {
|
|
9810
|
+
if (node._cameraRegistrySyncInFlight === syncPromise) node._cameraRegistrySyncInFlight = null
|
|
9811
|
+
})
|
|
9812
|
+
node._cameraRegistrySyncInFlight = syncPromise
|
|
9813
|
+
return syncPromise
|
|
9814
|
+
}
|
|
9815
|
+
node.refreshCameraAdapterRegistry = syncCameraAdapterRegistry
|
|
9816
|
+
|
|
9176
9817
|
const handleKnxAiConfirmationDecision = ({ msg, question, sessionId, decision }) => {
|
|
9177
9818
|
const pending = node._pendingKnxCommands.get(sessionId)
|
|
9178
9819
|
const language = pending && pending.language
|
|
@@ -9835,6 +10476,11 @@ module.exports = function (RED) {
|
|
|
9835
10476
|
node._conversationSessions = new Map()
|
|
9836
10477
|
node._chatContext = createEmptyKnxAiChatContext()
|
|
9837
10478
|
node._pendingKnxCommands = new Map()
|
|
10479
|
+
node._pendingCameraRequests.forEach(pending => {
|
|
10480
|
+
try { if (pending && pending.timer) clearTimeout(pending.timer) } catch (error) { /* ignore */ }
|
|
10481
|
+
})
|
|
10482
|
+
node._pendingCameraRequests = new Map()
|
|
10483
|
+
node._cameraWatchLastTriggered = new Map()
|
|
9838
10484
|
node._homeMemory = createEmptyKnxAiHomeMemory()
|
|
9839
10485
|
node._proactiveStates = new Map()
|
|
9840
10486
|
node._proactiveInFlight = new Set()
|
|
@@ -9881,14 +10527,18 @@ module.exports = function (RED) {
|
|
|
9881
10527
|
node._pendingKnxCommands.delete(sessionId)
|
|
9882
10528
|
updateStatus({ fill: 'blue', shape: 'ring', text: 'AI thinking...' })
|
|
9883
10529
|
try {
|
|
9884
|
-
|
|
10530
|
+
await syncCameraAdapterRegistry()
|
|
10531
|
+
const cameraChatAvailable = node._cameraAdapters.size > 0 || node._cameraCatalog.size > 0
|
|
10532
|
+
const ret = node.llmAllowKnxCommands || cameraChatAvailable
|
|
9885
10533
|
? await callConversationalLLM({
|
|
9886
10534
|
question,
|
|
9887
10535
|
sessionId,
|
|
9888
|
-
requireConfirmation: node.llmRequireCommandConfirmation
|
|
10536
|
+
requireConfirmation: node.llmRequireCommandConfirmation,
|
|
10537
|
+
allowKnxCommands: node.llmAllowKnxCommands
|
|
9889
10538
|
})
|
|
9890
10539
|
: await callLLM({ question, sessionId })
|
|
9891
10540
|
const preparedCommands = Array.isArray(ret.commands) ? ret.commands : []
|
|
10541
|
+
const preparedCameraActions = Array.isArray(ret.cameraActions) ? ret.cameraActions : []
|
|
9892
10542
|
const readCommands = preparedCommands.filter(command => command && command.event === 'GroupValue_Read')
|
|
9893
10543
|
const writeCommands = preparedCommands.filter(command => !command || command.event !== 'GroupValue_Read')
|
|
9894
10544
|
const language = resolveKnxAiLanguage(msg, node.llmDocsLanguage || 'en', question, ret.language)
|
|
@@ -9898,6 +10548,18 @@ module.exports = function (RED) {
|
|
|
9898
10548
|
node.llmRequireCommandConfirmation &&
|
|
9899
10549
|
writeCommands.length > 0
|
|
9900
10550
|
let content = ret.content
|
|
10551
|
+
const cameraActionResult = applyCameraActions({
|
|
10552
|
+
actions: preparedCameraActions,
|
|
10553
|
+
sessionId,
|
|
10554
|
+
inputMessage: msg,
|
|
10555
|
+
question,
|
|
10556
|
+
language,
|
|
10557
|
+
reply: content
|
|
10558
|
+
})
|
|
10559
|
+
if (cameraActionResult.additions.length) {
|
|
10560
|
+
content = [content].concat(cameraActionResult.additions).filter(Boolean).join('\n\n')
|
|
10561
|
+
}
|
|
10562
|
+
const deferCameraReply = cameraActionResult.deferredSnapshotReply && preparedCommands.length === 0
|
|
9901
10563
|
let commandsToEmit = preparedCommands
|
|
9902
10564
|
let confirmationRequest = null
|
|
9903
10565
|
if (awaitingConfirmation) {
|
|
@@ -9976,13 +10638,14 @@ module.exports = function (RED) {
|
|
|
9976
10638
|
sessionId,
|
|
9977
10639
|
commandCount: writeCommands.length,
|
|
9978
10640
|
readCount: readCommands.length,
|
|
10641
|
+
cameraActionCount: preparedCameraActions.length,
|
|
9979
10642
|
language,
|
|
9980
10643
|
awaitingConfirmation,
|
|
9981
10644
|
rejectedCommandCount: Array.isArray(ret.rejectedCommands) ? ret.rejectedCommands.length : 0
|
|
9982
10645
|
}
|
|
9983
10646
|
node._assistantLog.push(assistantEntry)
|
|
9984
10647
|
while (node._assistantLog.length > 50) node._assistantLog.shift()
|
|
9985
|
-
rememberConversationTurn({ sessionId, question, reply: content })
|
|
10648
|
+
if (!deferCameraReply) rememberConversationTurn({ sessionId, question, reply: content })
|
|
9986
10649
|
const replyMessage = buildKnxAiReplyMessage({
|
|
9987
10650
|
inputMessage: msg,
|
|
9988
10651
|
content,
|
|
@@ -9996,6 +10659,7 @@ module.exports = function (RED) {
|
|
|
9996
10659
|
operationCount: preparedCommands.length,
|
|
9997
10660
|
commandCount: writeCommands.length,
|
|
9998
10661
|
readCount: readCommands.length,
|
|
10662
|
+
cameraActionCount: preparedCameraActions.length,
|
|
9999
10663
|
readResults: readResultMetadata,
|
|
10000
10664
|
awaitingConfirmation,
|
|
10001
10665
|
confirmationExpiresAt: confirmationRequest ? confirmationRequest.expiresAt : 0,
|
|
@@ -10005,7 +10669,10 @@ module.exports = function (RED) {
|
|
|
10005
10669
|
},
|
|
10006
10670
|
summary: emittedReadCommands.length > 0 ? rebuildCachedSummaryNow() : ret.summary
|
|
10007
10671
|
})
|
|
10008
|
-
if (
|
|
10672
|
+
if (deferCameraReply) {
|
|
10673
|
+
// The matching camera provider returns the snapshot asynchronously;
|
|
10674
|
+
// its image (and optional visual analysis) becomes the chat reply.
|
|
10675
|
+
} else if (emittedReadCommands.length > 0) {
|
|
10009
10676
|
if (!sendKnxAiOutputs([null, null, replyMessage, null], msg)) return
|
|
10010
10677
|
} else if (!sendKnxAiOutputs([null, null, replyMessage, commandMessages.length ? commandMessages : null], msg)) {
|
|
10011
10678
|
return
|
|
@@ -10199,6 +10866,10 @@ module.exports = function (RED) {
|
|
|
10199
10866
|
return
|
|
10200
10867
|
}
|
|
10201
10868
|
if (!adaptedMessage) return
|
|
10869
|
+
const adaptedTopic = String(adaptedMessage.topic || '').toLocaleLowerCase()
|
|
10870
|
+
if (adaptedTopic === 'ask' || adaptedTopic === 'chat' || adaptedTopic === 'question' || adaptedTopic === 'prompt') {
|
|
10871
|
+
rememberChatSessionSource({ sessionId: resolveKnxAiSessionId(adaptedMessage), msg })
|
|
10872
|
+
}
|
|
10202
10873
|
const p = handleCommand(adaptedMessage)
|
|
10203
10874
|
if (p && typeof p.catch === 'function') {
|
|
10204
10875
|
p.catch((error) => {
|
|
@@ -10219,6 +10890,14 @@ module.exports = function (RED) {
|
|
|
10219
10890
|
if (node._busConnectionWatchTimer) clearInterval(node._busConnectionWatchTimer)
|
|
10220
10891
|
if (node._homeMemoryPeriodicTimer) clearInterval(node._homeMemoryPeriodicTimer)
|
|
10221
10892
|
if (node._proactiveCheckTimer) clearInterval(node._proactiveCheckTimer)
|
|
10893
|
+
if (node._cameraRegistrySyncTimer) clearInterval(node._cameraRegistrySyncTimer)
|
|
10894
|
+
node._cameraRegistrySyncTimer = null
|
|
10895
|
+
try { if (typeof node._cameraRegistryUnsubscribe === 'function') node._cameraRegistryUnsubscribe() } catch (error) { /* ignore */ }
|
|
10896
|
+
node._cameraRegistryUnsubscribe = null
|
|
10897
|
+
node._cameraProviderUnsubscribers.forEach(unsubscribe => {
|
|
10898
|
+
try { if (typeof unsubscribe === 'function') unsubscribe() } catch (error) { /* ignore */ }
|
|
10899
|
+
})
|
|
10900
|
+
node._cameraProviderUnsubscribers.clear()
|
|
10222
10901
|
if (node._homeMemoryWriteTimer) {
|
|
10223
10902
|
clearTimeout(node._homeMemoryWriteTimer)
|
|
10224
10903
|
node._homeMemoryWriteTimer = null
|
|
@@ -10227,6 +10906,12 @@ module.exports = function (RED) {
|
|
|
10227
10906
|
clearTimeout(node._chatContextWriteTimer)
|
|
10228
10907
|
node._chatContextWriteTimer = null
|
|
10229
10908
|
}
|
|
10909
|
+
if (node._pendingCameraRequests instanceof Map) {
|
|
10910
|
+
node._pendingCameraRequests.forEach(pending => {
|
|
10911
|
+
try { if (pending && pending.timer) clearTimeout(pending.timer) } catch (error) { /* ignore */ }
|
|
10912
|
+
})
|
|
10913
|
+
node._pendingCameraRequests.clear()
|
|
10914
|
+
}
|
|
10230
10915
|
persistHomeMemoryNow()
|
|
10231
10916
|
persistChatContextNow()
|
|
10232
10917
|
if (node._homeMemoryStorePath) {
|
|
@@ -10284,6 +10969,25 @@ module.exports = function (RED) {
|
|
|
10284
10969
|
node.sysLogger?.warn(`KNX AI history startup error: ${error.message || error}`)
|
|
10285
10970
|
}
|
|
10286
10971
|
|
|
10972
|
+
try {
|
|
10973
|
+
const cameraRegistry = getKnxAiCameraAdapterRegistry()
|
|
10974
|
+
node._cameraRegistryUnsubscribe = cameraRegistry.subscribe(() => {
|
|
10975
|
+
Promise.resolve(syncCameraAdapterRegistry({ force: true })).catch(error => {
|
|
10976
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter refresh error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10977
|
+
})
|
|
10978
|
+
})
|
|
10979
|
+
Promise.resolve(syncCameraAdapterRegistry({ force: true })).catch(error => {
|
|
10980
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter startup error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10981
|
+
})
|
|
10982
|
+
node._cameraRegistrySyncTimer = setInterval(() => {
|
|
10983
|
+
Promise.resolve(syncCameraAdapterRegistry()).catch(error => {
|
|
10984
|
+
try { node.sysLogger?.warn(`KNX AI camera adapter refresh error: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10985
|
+
})
|
|
10986
|
+
}, 30 * 1000)
|
|
10987
|
+
} catch (error) {
|
|
10988
|
+
try { node.sysLogger?.warn(`KNX AI camera registry unavailable: ${error.message || error}`) } catch (logError) { /* ignore */ }
|
|
10989
|
+
}
|
|
10990
|
+
|
|
10287
10991
|
if (node._homeMemoryPeriodicTimer) clearInterval(node._homeMemoryPeriodicTimer)
|
|
10288
10992
|
node._homeMemoryPeriodicTimer = setInterval(() => {
|
|
10289
10993
|
try { persistHomeMemoryNow() } catch (error) { /* persistHomeMemoryNow already guards */ }
|
|
@@ -10314,6 +11018,7 @@ module.exports = function (RED) {
|
|
|
10314
11018
|
|
|
10315
11019
|
module.exports.__test = {
|
|
10316
11020
|
bindSharedKnxAiState,
|
|
11021
|
+
applyKnxAiChatMediaPresetFallback,
|
|
10317
11022
|
buildKnxAiPackageNodeCatalog,
|
|
10318
11023
|
buildKnxAiConfirmationRequest,
|
|
10319
11024
|
buildKnxAiUniversalMessage,
|
|
@@ -10339,5 +11044,6 @@ module.exports.__test = {
|
|
|
10339
11044
|
resolveKnxAiSessionId,
|
|
10340
11045
|
releaseSharedKnxAiState,
|
|
10341
11046
|
safeKnxAiSend,
|
|
11047
|
+
summarizeDetectedKnxAiCameraAdapters,
|
|
10342
11048
|
validateKnxAiPayloadForDpt
|
|
10343
11049
|
}
|