@fugood/bricks-project 2.24.0-beta.14 → 2.24.0-beta.16

package/compile/action-name-map.ts

@@ -772,6 +772,20 @@ export const templateActionNameMap = {
       documents: 'GENERATOR_RERANKER_DOCUMENTS',
     },
   },
+  GENERATOR_MLX_LLM: {
+    GENERATOR_MLX_LLM_LOAD_MODEL: {
+      modelId: 'GENERATOR_MLX_LLM_MODEL_ID',
+      vlm: 'GENERATOR_MLX_LLM_VLM',
+    },
+    GENERATOR_MLX_LLM_COMPLETION: {
+      messages: 'GENERATOR_MLX_LLM_MESSAGES',
+      tools: 'GENERATOR_MLX_LLM_TOOLS',
+      maxTokens: 'GENERATOR_MLX_LLM_MAX_TOKENS',
+      temperature: 'GENERATOR_MLX_LLM_TEMPERATURE',
+      topP: 'GENERATOR_MLX_LLM_TOP_P',
+      repetitionPenalty: 'GENERATOR_MLX_LLM_REPETITION_PENALTY',
+    },
+  },
   GENERATOR_QNN_LLM: {
     GENERATOR_QNN_LLM_PROCESS: {
       prompt: 'GENERATOR_QNN_LLM_PROMPT',

package/compile/index.ts

@@ -670,6 +670,8 @@ export const compile = async (app: Application) => {
                 buildList(item, index, 'brickList'),
               )
               property.brickList = brickList
+            } else if (!brickItems.brickList) {
+              property.brickList = []
             } else {
               // Not supported Data for brickList
               throw new TypeError('Not supported Data for brickList directly')
@@ -679,6 +681,8 @@ export const compile = async (app: Application) => {
                 buildList(item, index, 'brickDetails'),
               )
               property.brickDetails = brickDetails
+            } else if (!brickItems.brickDetails) {
+              property.brickDetails = []
             } else {
               // Not supported Data for brickList
               throw new TypeError('Not supported Data for brickList directly')

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.24.0-beta.14",
+  "version": "2.24.0-beta.16",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit",
     "build": "bun scripts/build.js"
   },
   "dependencies": {
-    "@fugood/bricks-cli": "^2.24.0-beta.14",
+    "@fugood/bricks-cli": "^2.24.0-beta.15",
     "@huggingface/gguf": "^0.3.2",
     "@iarna/toml": "^3.0.0",
     "@modelcontextprotocol/sdk": "^1.15.0",
@@ -24,5 +24,5 @@
   "peerDependencies": {
     "oxfmt": "^0.36.0"
   },
-  "gitHead": "f24caa2ad21dbc7746888e927f6c8275578138aa"
+  "gitHead": "b024e1980b2c25797ae26e9b973497e3ee724265"
 }

package/skills/bricks-project/rules/buttress.md

@@ -14,8 +14,9 @@ When mobile devices or embedded systems lack hardware for local AI inference (LL
 
 ## Supported Generators
 
-- [LLM (GGML)](generator_LLM) - Local Large Language Model inference
-- [Speech-to-Text (GGML)](generator_SpeechInference) - Local Speech-to-Text inference
+- LLM (GGML) (LlmMlx.ts) - Local Large Language Model inference with GGML
+- LLM (MLX) (LlmGgml.ts) - Local Large Language Model inference with MLX
+- Speech-to-Text (GGML) (SpeechToTextGgml.ts) - Local Speech-to-Text inference with GGML
 
 ## Client Configuration
 
@@ -50,10 +51,12 @@ const llmGenerator: GeneratorLLM = {
   property: {
     modelUrl: 'https://huggingface.co/ggml-org/gemma-3-12b-it-qat-GGUF/resolve/main/gemma-3-12b-it-qat-q4_0.gguf',
     contextSize: 8192,
-    buttressEnabled: true,
-    buttressUrl: 'http://192.168.1.100:2080',
-    buttressFallbackType: 'use-local',
-    buttressStrategy: 'prefer-best',
+    buttressConnectionSettings: {
+      enabled: true,
+      url: 'http://192.168.1.100:2080',
+      fallbackType: 'use-local',
+      strategy: 'prefer-best',
+    },
   },
   events: {},
   switches: [],

package/tools/mcp-tools/huggingface.ts

@@ -102,6 +102,7 @@ type HFSibling = {
   rfilename: string
   size?: number
   lfs?: { sha256?: string }
+  blobId?: string
 }
 
 type HFModel = {
@@ -132,8 +133,9 @@ type GeneratorType =
   | 'GeneratorOnnxLLM'
   | 'GeneratorOnnxSTT'
   | 'GeneratorTTS'
+  | 'GeneratorMlxLLM'
 
-type ModelKind = 'gguf' | 'onnx'
+type ModelKind = 'gguf' | 'onnx' | 'mlx'
 
 interface GeneratorConfig {
   modelKind: ModelKind
@@ -219,6 +221,11 @@ const generatorConfigs: Record<GeneratorType, GeneratorConfig> = {
     filter: 'onnx,text-to-speech',
     hasValidStructure: hasValidOnnxStructure,
   },
+  GeneratorMlxLLM: {
+    modelKind: 'mlx',
+    filter: 'mlx',
+    taskFilter: supportedLlmTasks,
+  },
 }
 
 const searchHFModels = async (filter: string, search?: string, limit = 50): Promise<HFModel[]> => {
@@ -276,6 +283,7 @@ export function register(server: McpServer) {
           'GeneratorOnnxLLM',
           'GeneratorOnnxSTT',
           'GeneratorTTS',
+          'GeneratorMlxLLM',
         ])
         .describe('Generator type to search models for')
         .default('GeneratorLLM'),
@@ -407,6 +415,7 @@ export function register(server: McpServer) {
           'GeneratorOnnxLLM',
           'GeneratorOnnxSTT',
           'GeneratorTTS',
+          'GeneratorMlxLLM',
         ])
         .describe('Generator type for model selection')
         .default('GeneratorLLM'),
@@ -489,7 +498,107 @@ export function register(server: McpServer) {
 
           // Return JSON for huggingface_select to allow direct parsing by consumers
           return {
-            content: [{ type: 'text', text: JSON5.stringify(result, null, 2) }],
+            content: [
+              {
+                type: 'text',
+                text: `${JSON5.stringify(result, null, 2)} // Please use _hfRepoInfo undocumented field with @ts-ignore`,
+              },
+            ],
+          }
+        }
+
+        // Handle MLX models (repo-level, like ONNX)
+        if (config.modelKind === 'mlx') {
+          // Fetch config.json for model architecture metadata
+          const headers: Record<string, string> = {}
+          if (HF_TOKEN) headers['Authorization'] = `Bearer ${HF_TOKEN}`
+          let mlxModelConfig: Record<string, any> | null = null
+          try {
+            const configRes = await fetch(
+              `https://huggingface.co/${modelId}/resolve/main/config.json`,
+              { headers },
+            )
+            if (configRes.ok) mlxModelConfig = await configRes.json()
+          } catch {
+            // Non-critical
+          }
+
+          const modelType =
+            mlxModelConfig?.model_type || details.config?.model_type || details.cardData?.model_type
+
+          // Build MLX model metadata from config.json (for hardware guardrails)
+          const cfg = mlxModelConfig || ({} as Record<string, any>)
+          const textCfg = cfg.text_config || cfg
+          const numHeads = textCfg.num_attention_heads || textCfg.n_heads || 0
+          const hiddenSize = textCfg.hidden_size || textCfg.dim || 0
+          const kvLoraRank = textCfg.kv_lora_rank || 0
+          const quant = cfg.quantization || cfg.quantization_config || null
+
+          // Sum safetensors/npz file sizes for model weight bytes
+          const modelBytes = siblings
+            .filter((f) => /\.(safetensors|npz)$/.test(f.rfilename))
+            .reduce((sum, f) => sum + (f.size ?? 0), 0)
+
+          // Build _mlxDownloadFiles list (safetensors, json, jinja, tokenizer.model)
+          const mlxDownloadFiles = siblings
+            .filter(
+              (f) =>
+                f.rfilename.endsWith('.safetensors') ||
+                f.rfilename.endsWith('.json') ||
+                f.rfilename.endsWith('.jinja') ||
+                f.rfilename === 'tokenizer.model',
+            )
+            .map((f) => ({
+              url: `https://huggingface.co/${modelId}/resolve/main/${f.rfilename}?download=true`,
+              filename: `${modelId.replace('/', '-')}/${f.rfilename}`,
+              hash_type: f.lfs ? 'sha256' : f.blobId ? 'sha1' : undefined,
+              sha256: f.lfs?.sha256,
+              sha1: f.lfs ? undefined : f.blobId,
+            }))
+
+          const result = {
+            modelId,
+            modelType,
+            _mlxDownloadFiles: mlxDownloadFiles,
+            _hfRepoInfo: {
+              repo: modelId,
+              model: {
+                id: details.id,
+                downloads: details.downloads,
+                likes: details.likes,
+                author: details.author,
+              },
+              generatorType,
+              modelType,
+              ...(modelBytes > 0 && { size: modelBytes }),
+              ...(mlxModelConfig && {
+                mlxModelMetadata: {
+                  hiddenSize,
+                  numLayers: textCfg.num_hidden_layers || textCfg.n_layers || 0,
+                  numHeads,
+                  numKvHeads: textCfg.num_key_value_heads ?? numHeads,
+                  headDim:
+                    textCfg.head_dim ||
+                    textCfg.v_head_dim ||
+                    (numHeads > 0 && hiddenSize > 0 ? hiddenSize / numHeads : 0),
+                  maxCtx: textCfg.max_position_embeddings || 0,
+                  kvLoraRank,
+                  qkRopeHeadDim: textCfg.qk_rope_head_dim || 0,
+                  isMLA: kvLoraRank > 0,
+                  quantBits: quant?.bits || null,
+                  quantGroupSize: quant?.group_size || null,
+                },
+              }),
+            },
+          }
+
+          return {
+            content: [
+              {
+                type: 'text',
+                text: `${JSON5.stringify(result, null, 2)} // IMPORTANT: Use _mlxDownloadFiles / _hfRepoInfo undocumented fields with @ts-ignore`,
+              },
+            ],
           }
         }
 
@@ -638,7 +747,7 @@ export function register(server: McpServer) {
             content: [
               {
                 type: 'text',
-                text: `${JSON5.stringify(result, null, 2)} // Please use _hfRepoInfo undocumented field with @ts-ignore`,
+                text: `${JSON5.stringify(result, null, 2)} // IMPORTANT: Use _hfRepoInfo undocumented field with @ts-ignore`,
               },
             ],
           }

package/types/bricks/Camera.ts

@@ -188,7 +188,7 @@ Default property:
   }
 }
 
-/* Camera view brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378589-camera)) */
+/* Camera view brick */
 export type BrickCamera = Brick &
   BrickCameraDef & {
     templateKey: 'BRICK_CAMERA'

package/types/bricks/Image.ts

@@ -87,7 +87,7 @@ Default property:
   }
 }
 
-/* Image brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378576-image)) */
+/* Image brick */
 export type BrickImage = Brick &
   BrickImageDef & {
     templateKey: 'BRICK_IMAGE'

package/types/bricks/Lottie.ts

@@ -137,7 +137,7 @@ Default property:
   }
 }
 
-/* Lottie Adobe After Effects animations brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378583-lottie)) */
+/* Lottie Adobe After Effects animations brick */
 export type BrickLottie = Brick &
   BrickLottieDef & {
     templateKey: 'BRICK_LOTTIE'

package/types/bricks/Maps.ts

@@ -231,7 +231,7 @@ Default property:
   }
 }
 
-/* Maps brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/maps)) */
+/* Maps brick */
 export type BrickMaps = Brick &
   BrickMapsDef & {
     templateKey: 'BRICK_MAPS'

package/types/bricks/QrCode.ts

@@ -90,7 +90,7 @@ Default property:
   }
 }
 
-/* QRCode brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378579-qr-code)) */
+/* QRCode brick */
 export type BrickQrcode = Brick &
   BrickQrcodeDef & {
     templateKey: 'BRICK_QRCODE'

package/types/bricks/Rect.ts

@@ -88,7 +88,7 @@ Default property:
   }
 }
 
-/* Rect brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378573-rect)) */
+/* Rect brick */
 export type BrickRect = Brick &
   BrickRectDef & {
     templateKey: 'BRICK_RECT'

package/types/bricks/Slideshow.ts

@@ -168,7 +168,7 @@ Default property:
   }
 }
 
-/* Slideshow brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378580-slideshow)) */
+/* Slideshow brick */
 export type BrickSlideshow = Brick &
   BrickSlideshowDef & {
     templateKey: 'BRICK_SLIDESHOW'

package/types/bricks/Text.ts

@@ -121,7 +121,7 @@ Default property:
   }
 }
 
-/* Text brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378574-text)) */
+/* Text brick */
 export type BrickText = Brick &
   BrickTextDef & {
     templateKey: 'BRICK_TEXT'

package/types/bricks/TextInput.ts

@@ -209,7 +209,7 @@ Default property:
   }
 }
 
-/* Text Input brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378575-text-input)) */
+/* Text Input brick */
 export type BrickTextInput = Brick &
   BrickTextInputDef & {
     templateKey: 'BRICK_TEXT_INPUT'

package/types/bricks/Video.ts

@@ -148,7 +148,7 @@ Default property:
   }
 }
 
-/* Video brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378577-video)) */
+/* Video brick */
 export type BrickVideo = Brick &
   BrickVideoDef & {
     templateKey: 'BRICK_VIDEO'

package/types/bricks/WebView.ts

@@ -141,7 +141,7 @@ Default property:
   }
 }
 
-/* WebView brick ([Tutorial](https://intercom.help/bricks-dag-inc/articles/5378588-web-view)) */
+/* WebView brick */
 export type BrickWebView = Brick &
   BrickWebViewDef & {
     templateKey: 'BRICK_WEBVIEW'

package/types/generators/Assistant.ts

@@ -434,7 +434,7 @@ Default property:
       | DataLink
     /* Whether to cache messages */
     cacheMessages?: boolean | DataLink
-    /* LLM Generator (Supports `LLM (GGML)` and `OpenAI LLM` generators) */
+    /* LLM Generator (Supports `LLM (GGML)`, `LLM (MLX)`, and `OpenAI LLM` generators) */
     llmGeneratorId?: string | DataLink | (() => Generator)
     /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use.
 

package/types/generators/LlmMlx.ts

@@ -0,0 +1,210 @@
+/* Auto generated by build script */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load model */
+export type GeneratorMlxLLMActionLoadModel = ActionWithParams & {
+  __actionName: 'GENERATOR_MLX_LLM_LOAD_MODEL'
+  params?: Array<
+    | {
+        input: 'modelId'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'vlm'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Run text completion */
+export type GeneratorMlxLLMActionCompletion = ActionWithParams & {
+  __actionName: 'GENERATOR_MLX_LLM_COMPLETION'
+  params?: Array<
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxTokens'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'temperature'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'topP'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'repetitionPenalty'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Stop text completion */
+export type GeneratorMlxLLMActionStopCompletion = Action & {
+  __actionName: 'GENERATOR_MLX_LLM_STOP_COMPLETION'
+}
+
+/* Release model context */
+export type GeneratorMlxLLMActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_MLX_LLM_RELEASE_CONTEXT'
+}
+
+interface GeneratorMlxLLMDef {
+  /*
+Default property:
+{
+  "init": false,
+  "modelId": "mlx-community/Qwen3-4B-4bit",
+  "vlm": false,
+  "completionMessages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "Hello"
+    }
+  ],
+  "completionMaxTokens": 1024,
+  "completionTemperature": 0.6,
+  "completionTopP": 1,
+  "completionRepetitionContextSize": 20
+}
+  */
+  property?: {
+    /* Initialize model on generator init */
+    init?: boolean | DataLink
+    /* HuggingFace model ID or local path to model directory
+ e.g. "mlx-community/Qwen3-4B-4bit" */
+    modelId?: string | DataLink
+    /* Enable Vision Language Model (VLM) mode */
+    vlm?: boolean | DataLink
+    /* Chat messages (if first message has role 'system', it will be used as system prompt) */
+    completionMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Maximum tokens to generate */
+    completionMaxTokens?: number | DataLink
+    /* Temperature (0.0 to 2.0) */
+    completionTemperature?: number | DataLink
+    /* Top P sampling */
+    completionTopP?: number | DataLink
+    /* Repetition penalty factor */
+    completionRepetitionPenalty?: number | DataLink
+    /* Number of tokens to consider for repetition penalty */
+    completionRepetitionContextSize?: number | DataLink
+    /* Maximum KV cache size (context window). Uses rotating cache to limit memory.
+ Smaller values (e.g. 512) use less memory but lower quality.  
+ Larger values (e.g. 4096) use more memory but better quality.  
+ Leave empty for unlimited (model default). */
+    contextSize?: number | DataLink
+    /* Enable thinking mode (model-dependent, e.g. Qwen3).
+ When enabled, passes enable_thinking=true to the chat template.  
+ The model may output reasoning in &lt;think&gt; tags which will be extracted as reasoning_content. */
+    completionEnableThinking?: boolean | DataLink
+    /* Additional keyword arguments for chat template (object) */
+    completionChatTemplateKwargs?: {} | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    completionTools?: Array<{} | DataLink> | DataLink
+    /* Buttress connection settings for remote inference */
+    buttressConnectionSettings?:
+      | DataLink
+      | {
+          enabled?: boolean | DataLink
+          url?: string | DataLink
+          fallbackType?: 'use-local' | 'no-op' | DataLink
+          strategy?: 'prefer-local' | 'prefer-buttress' | 'prefer-best' | DataLink
+        }
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Error event */
+    onError?: Array<EventAction>
+    /* Completion streaming event (emitted for each token) */
+    onCompletion?: Array<EventAction>
+    /* Completion finished event */
+    onCompletionFinished?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Model load progress (0-1) */
+    loadProgress?: () => Data
+    /* Whether the model is evaluating */
+    isEvaluating?: () => Data
+    /* Completion result */
+    completionResult?: () => Data
+    /* Last token */
+    completionLastToken?: () => Data
+  }
+}
+
+/* On-device LLM inference using Apple MLX framework on iOS/tvOS
+
+ ## Features  
+ - Powered by MLX (Apple's ML framework optimized for Apple Silicon)  
+ - Download models directly from HuggingFace Hub  
+ - Streaming token generation  
+ - Supports LLM and VLM (Vision Language Models)  
+ - Requires iOS 17+ or tvOS 17+ */
+export type GeneratorMlxLLM = Generator &
+  GeneratorMlxLLMDef & {
+    templateKey: 'GENERATOR_MLX_LLM'
+    switches?: Array<
+      SwitchDef &
+        GeneratorMlxLLMDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'loadProgress'
+                    | 'isEvaluating'
+                    | 'completionResult'
+                    | 'completionLastToken'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/index.ts

@@ -44,6 +44,7 @@ export * from './RealtimeTranscription'
 export * from './LlmGgml'
 export * from './TextToSpeechGgml'
 export * from './RerankerGgml'
+export * from './LlmMlx'
 export * from './LlmQualcommAiEngine'
 export * from './LlmOpenAiCompat'
 export * from './TextToSpeechOpenAiLike'

package/types/system.ts

@@ -276,7 +276,7 @@ export type SystemActionPopupReset = ActionWithParams & {
   >
 }
 
-/* Take screenshot for current subspace */
+/* Take screenshot for current subspace or the full viewport (including portal subspaces) */
 export type SystemActionTakeScreenshot = ActionWithParams & {
   __actionName: 'TAKE_SCREENSHOT'
   params?: Array<

package/utils/event-props.ts

@@ -905,6 +905,22 @@ export const templateEventPropsMap = {
       'GENERATOR_RERANKER_ERROR', // type: string
     ],
   },
+  GENERATOR_MLX_LLM: {
+    onContextStateChange: [
+      'GENERATOR_MLX_LLM_CONTEXT_STATE', // type: string
+    ],
+    onError: [
+      'GENERATOR_MLX_LLM_ERROR', // type: string
+    ],
+    onCompletion: [
+      'GENERATOR_MLX_LLM_COMPLETION_TOKEN', // type: string
+      'GENERATOR_MLX_LLM_COMPLETION_RESULT', // type: string
+    ],
+    onCompletionFinished: [
+      'GENERATOR_MLX_LLM_COMPLETION_RESULT', // type: string
+      'GENERATOR_MLX_LLM_COMPLETION_RESULT_DETAILS', // type: object
+    ],
+  },
   GENERATOR_QNN_LLM: {
     onContextStateChange: [
       'GENERATOR_QNN_LLM_CONTEXT_STATE', // type: string