@thinkingai/ae-cli 6.1.27 → 6.1.28

package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/build-project-semantic-wiki.mjs

@@ -7,6 +7,7 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { precompiledSource, normalizeSourceForHash } from './precompiled-source.mjs';
+import { reportContract } from './execution-contract.mjs';
 
 const require = createRequire(import.meta.url);
 const archiverModule = require('archiver');
@@ -259,6 +260,7 @@ try {
         personal_semantic_integrations: planModel.personalSemanticIntegrations.length,
         blocked_assets: blockedCount, unknown_sql_reports: unknownSqlCount },
       quality_warnings: [...asArray(corpus.warnings),
+        ...planModel.cards.filter(card => !card.routing).map(card => `Recall card ${card.card_id} has legacy routing; precise intent branches require semantic review.`),
         ...(sourceProjectRefs.filter((row) => Number(row.source_project_id) !== semanticProjectId).length
           ? [`${sourceProjectRefs.filter((row) => Number(row.source_project_id) !== semanticProjectId).length} packaged asset reference(s) mention another project: ${sourceProjectRefs.filter((row) => Number(row.source_project_id) !== semanticProjectId).slice(0, 5).map((row) => `${row.resource_type}:${row.resource_key}->${row.source_project_id}`).join(', ')}`] : []),
         ...(unknownSqlCount ? [`${unknownSqlCount} SQL reports have unknown semantics and are blocked.`] : []),
@@ -350,6 +352,22 @@ function validateSemanticPlan() {
               fail(`Recall card ${cardId} routing branch references an asset outside preferred/fallback refs.`);
             }
           }
+          const combination = asObject(branch.combination);
+          if (branch.combination != null) {
+            if (!['compatible', 'incompatible', 'unknown'].includes(clean(combination.compatibility))) fail(`Recall card ${cardId} has invalid combination compatibility.`);
+            if (!clean(combination.reason)) fail(`Recall card ${cardId} combination requires an evidence-based reason.`);
+            if (combination.compatibility === 'compatible'
+              && (!asArray(combination.join_keys).length || !clean(combination.time_alignment))) {
+              fail(`Recall card ${cardId} compatible combination requires join_keys and time_alignment.`);
+            }
+            for (const ref of asArray(branch.asset_refs).filter(ref => ref.resource_type === 'report')) {
+              const summary = asArray(semanticPlan.sql_report_semantics).find(row => clean(row.report_id) === clean(ref.resource_key));
+              if (summary && combination.compatibility === 'compatible'
+                && asArray(combination.join_keys).some(key => !asArray(summary.output_fields).some(field => clean(field.name) === clean(key)))) {
+                fail(`Recall card ${cardId} join key is not an output field of report ${ref.resource_key}.`);
+              }
+            }
+          }
         }
       }
       for (const ref of asArray(card.excluded_asset_refs)) {
@@ -691,6 +709,7 @@ function writeMergedAsset(wikiRoot, directory, asset, authority, extra = '') {
   const id = clean(asset.resource_key);
   const source = readText(path.join(buildIr, directory, fileName(directory.slice(0, -1), id)))
     .replaceAll('../appendices/', '../technical-appendices/')
+    .replace(/^- 定义状态:\s*(valid|unknown|conflict)\s*$/m, `- 定义状态: ${authority.definition_state}`)
     .replace(/- 源详情: `([^`]+)`/g, '- 资产包源定位（精确证据已内嵌）: `$1`');
   const brief = readText(path.join(buildIr, 'briefs', directory, fileName(directory.slice(0, -1), id)));
   const type = directory === 'dashboards' ? 'project-dashboard' : directory === 'reports' ? 'project-report' : 'project-reusable-metric';
@@ -755,8 +774,11 @@ ${asArray(values).map((ref) => {
   const routing = asObject(card.routing);
   const branches = asArray(routing.branches).map((branch) => {
     const item = asObject(branch);
-    const targets = asArray(item.asset_refs).map(recallAssetLabel).join('、') || '回到业务域核对资产';
-    return `- 当${clean(item.when)}：使用 ${targets}。${clean(item.reason)}`;
+    const targets = asArray(item.asset_refs).map(ref => `[${escapeLink(recallAssetLabel(ref))}](${assetPath(clean(ref.resource_type), clean(ref.resource_key))})`).join('、') || '回到业务域核对资产';
+    const combination = asObject(item.combination);
+    const combinationNote = asArray(item.asset_refs).filter(ref => ref.resource_type !== 'dashboard').length > 1
+      ? ` Combination: ${clean(combination.compatibility) || 'unknown'}; ${clean(combination.reason) || 'Verify common population, stable identity, grain and time before combining.'}${asArray(combination.join_keys).length ? ` Keys: ${asArray(combination.join_keys).join(', ')}.` : ''}${clean(combination.time_alignment) ? ` Time: ${clean(combination.time_alignment)}.` : ''}${asArray(combination.missing_coverage).length ? ` Gaps: ${asArray(combination.missing_coverage).join('; ')}.` : ''}` : '';
+    return `- 当${clean(item.when)}：使用 ${targets}。${clean(item.reason)}${combinationNote}`;
   });
   return `---
 type: project-semantic-recall-card
@@ -801,6 +823,8 @@ ${refs(card.fallback_asset_refs, '备选资产')}
 
 ${refs(card.excluded_asset_refs, '排除资产')}
 
+${renderRecallEvidence(card)}
+
 ## 实时执行边界
 
 - 是否需要执行实时数据：${Boolean(card.requires_live_execution)}
@@ -816,11 +840,48 @@ function recallAssetLabel(ref) {
     : type === 'report' ? reportById.get(key)
       : type === 'metric' ? metricById.get(key)
         : null;
-  const title = assetReadableTitle({ ...asObject(row), title: ref.display_name || ref.title || row?.title, resource_key: key }, type);
+  const title = assetReadableTitle({ ...asObject(row), title: row?.title || ref.display_name || ref.title, resource_key: key }, type);
   const identity = `${type}:${key}`;
   return title && title !== key ? `${title}（${identity}）` : identity;
 }
 
+function renderRecallEvidence(card) {
+  const refs = [...asArray(card.preferred_asset_refs), ...asArray(card.fallback_asset_refs),
+    ...asArray(asObject(card.routing).branches).flatMap(branch => asArray(branch.asset_refs))];
+  const seen = new Set();
+  const lines = [];
+  for (const ref of refs) {
+    const identity = `${ref.resource_type}:${ref.resource_key}`;
+    if (seen.has(identity) || !['report', 'metric'].includes(ref.resource_type)) continue;
+    seen.add(identity);
+    const report = ref.resource_type === 'report' ? reportById.get(clean(ref.resource_key)) : null;
+    const summary = report ? planModel.sqlByReportId.get(clean(ref.resource_key)) : null;
+    const facts = report ? reportContract(report) : null;
+    const parameters = summary ? asArray(summary.input_parameters).map(item => `${clean(item.name)}: ${clean(item.meaning)}`) : [];
+    const measures = summary ? [] : asArray(report?.metric_occurrences).map(item =>
+      [clean(item.display_name || item.event_title || item.event_name), clean(item.aggregation_name || item.aggregation_code)].filter(Boolean).join(': '));
+    const savedScope = !summary && facts && Object.keys(facts.time).length ? ` Saved time scope: ${JSON.stringify(facts.time)}.` : '';
+    const reportSource = summary ? readText(path.join(buildIr, 'reports', fileName('report', clean(ref.resource_key)))) : '';
+    const outputs = summary ? asArray(summary.output_fields).map(item => {
+      const name = clean(item.name);
+      const unverified = reportSource.split('\n').some(line => line.startsWith(`- \`${name}\``) && line.includes('output identity unverified'));
+      if (unverified && asArray(asObject(card.routing).branches).some(branch =>
+        branch.combination?.compatibility === 'compatible'
+        && asArray(branch.combination.join_keys).includes(name)
+        && asArray(branch.asset_refs).some(asset => asset.resource_type === 'report' && clean(asset.resource_key) === clean(ref.resource_key)))) {
+        fail(`Recall card ${card.card_id} join key has unverified output identity in report ${ref.resource_key}: ${name}.`);
+      }
+      return `${name}${unverified ? ' (identity unknown)' : ''}`;
+    }) : [];
+    const reminders = summary ? [
+      ...asArray(summary.calculation_rules).slice(0, 1).map(item => clean(item.meaning)),
+      ...['time_semantics', 'coverage_gaps'].flatMap(key => asArray(summary[key]).map(item => clean(item.meaning))),
+    ] : [];
+    lines.push(`- [${escapeLink(recallAssetLabel(ref))}](${assetPath(ref.resource_type, clean(ref.resource_key))}): ${clean(summary?.business_purpose) || clean(ref.reason)} Grain: ${clean(summary?.statistical_grain) || (facts ? JSON.stringify(facts.groups) : 'verify the asset definition')}.${parameters.length ? ` Parameters: ${parameters.join('; ')}.` : ''}${outputs.length ? ` Outputs: ${outputs.join(', ')}.` : ''}${measures.length ? ` Measures (verify result column identities): ${measures.join('; ')}.` : ''}${savedScope}${reminders.length ? ` Reminders: ${reminders.join('; ')}.` : ''}`);
+  }
+  return lines.length ? `## Minimum asset evidence\n\n${lines.join('\n')}\n\nVerify only dashboard notes, inherited filters or timezone settings that affect the chosen route. Asset visibility does not establish runtime data permissions.` : '';
+}
+
 function renderRecallCardIndex(cards) {
   return `# 场景召回卡\n\n先按问法、对象、粒度和参数选择匹配的卡；卡内按条件分支找到最小证据并在满足停止条件后停止。无匹配卡时，回到[业务领域](../domains/index.md)，按对象、粒度、过滤和时间筛选域内资产；仍无可用资产时说明具体缺口。\n\n${cards.map((card) => `- [${escapeLink(card.intent || card.card_id)}](${fileName('recall-card', card.card_id)})（${clean(card.domain_title)}）`).join('\n')}\n`;
 }
@@ -1481,7 +1542,15 @@ function semanticPlanFragmentRefsForSource(identity, rel) {
   if (rel === 'wiki/project-overview.md') refs.add('project:model');
   if (sourceKind === 'domain' && assetKey) refs.add(`domain:${assetKey}`);
   if (sourceKind === 'appendix' && assetKey) refs.add(`appendix:${assetKey}`);
-  if (sourceKind === 'recall-card' && assetKey) refs.add(`recall-card:${assetKey}`);
+  if (sourceKind === 'recall-card' && assetKey) {
+    refs.add(`recall-card:${assetKey}`);
+    const card = planModel.cards.find(card => clean(card.card_id) === assetKey);
+    for (const ref of [...asArray(card?.preferred_asset_refs), ...asArray(card?.fallback_asset_refs)]) {
+      if (ref.resource_type === 'report' && planModel.sqlByReportId.has(clean(ref.resource_key))) {
+        refs.add(`report:${clean(ref.resource_key)}:sql_semantics`);
+      }
+    }
+  }
   if (sourceKind === 'personal-semantic' && assetKey) {
     refs.add(`personal-semantic-integration:${assetKey}`);
   }

package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/default-compile-schema.md

@@ -1,6 +1,6 @@
 # 项目语义知识库编译准则
 
-预置版本：1.1。导航分类版本：2.1。
+预置版本：1.2。导航分类版本：2.1。
 
 ## 总体要求
 
@@ -27,6 +27,10 @@
 
 不得将安全状态当作过时占位信息而忽略。保留认证状态、定义有效性、冲突、运行时策略和召回限制。已认证的资产仍可能存在定义未知的情况。仅作为依赖的资产可以解释定义、依赖关系和冲突，但不能独立证明业务状态或当前结果。遵守平台访问权限；权限不足时，不得改用其他项目或扩大范围来替代。
 
+源文件中的 `Execution contract`、计算证据及组合适用性是需要保留的口径事实。完整保留条件树的 AND/OR 与层级、公式各操作数的过滤和时间窗、去重实体、单位与取整、数组展开、模型窗口、人群条件、聚合参数和保存设置。参数身份、输出列名、显示名和解释分别保留，不互相替换。保存默认值与本次执行覆盖值不能混为一谈；行为日期不证明维表或用户表的历史状态。
+
+召回卡内已存在的精确资产路线、参数角色、组合键和覆盖缺口应直接保留，不退回宽泛目录。组合为 `unknown` 或 `incompatible` 时，不改写为可以直接关联。缺行、零值、空数组和未知分母保持区别；补证仅针对尚未确认的候选范围，不假定用户有底层 SQL 权限。计算证据与源定义不一致时记录冲突，不临场补写公式、修正源 SQL 或套用固定日期与阈值。
+
 ## 权威性与补充知识
 
 - 资产定义、治理事实和实时结果保留各自的权威性。决策或纠正仅在明确的目标、内容、日期、状态和适用范围内覆盖其他材料；时间较新本身不足以构成覆盖依据。

package/skills/ae-analysis/scripts/project-semantic-knowledge-wiki/execution-contract.mjs

@@ -0,0 +1,173 @@
+// Structural facts only. Business meanings and asset routes remain Agent-authored.
+const obj = value => value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+const list = value => Array.isArray(value) ? value : [];
+const valueOf = value => {
+  if (typeof value !== 'string') return value;
+  try { return JSON.parse(value); } catch { return value; }
+};
+const present = value => value !== undefined && value !== null && value !== '';
+const pick = (source, keys) => Object.fromEntries(keys.filter(key => present(source[key])).map(key => [key, source[key]]));
+const displayKeys = new Set(['display_name', 'eventNameDisplay', 'eventDesc', 'analysisDesc', 'quotaDesc', 'columnDesc', 'eventUuid', 'uuid']);
+
+export function semanticValue(value) {
+  if (Array.isArray(value)) return value.map(semanticValue);
+  if (!value || typeof value !== 'object') return value;
+  return Object.fromEntries(Object.keys(value).filter(key => !displayKeys.has(key)).sort()
+    .map(key => [key, semanticValue(value[key])]));
+}
+
+export function semanticTime(value) {
+  const time = { ...obj(value) };
+  if (present(time.recentDay ?? time.recent_day)) {
+    for (const key of ['startTime', 'endTime', 'start_time', 'end_time']) delete time[key];
+  }
+  return time;
+}
+
+export function logicalRelation(value) {
+  const relation = String(value ?? '').toLowerCase();
+  return ({ '1': 'AND', '0': 'OR', and: 'AND', or: 'OR' })[relation] || `UNKNOWN(${relation || 'unspecified'})`;
+}
+
+export function conditionTree(value, relation) {
+  if (Array.isArray(value)) return { relation: logicalRelation(relation), items: value.map(item => conditionTree(item)) };
+  const source = obj(value);
+  const nested = source.filts ?? source.items ?? source.filters;
+  if (nested != null) {
+    if (!Array.isArray(nested)) return { unknown: true, raw: source };
+    return { relation: logicalRelation(source.relation ?? source.junctionKind ?? relation), items: nested.map(item => conditionTree(item)) };
+  }
+  if (!Object.keys(source).length) return { unknown: true };
+  return { predicate: source };
+}
+
+export function renderConditionTree(value, leaf) {
+  if (value.predicate) return leaf(value.predicate);
+  if (value.unknown) return 'Unknown predicate; verify the source definition';
+  if (!value.items.length) return 'No conditions in this scope';
+  if (value.items.length === 1) return renderConditionTree(value.items[0], leaf);
+  return `(${value.items.map(item => renderConditionTree(item, leaf)).join(` ${value.relation} `)})`;
+}
+
+export function metricContract(item) {
+  const formula = obj(item.formula_definition ?? item.formula);
+  return {
+    ...pick(item, ['event_name', 'aggregation_code', 'measure_key', 'formula_expression', 'entity_grain', 'aggregation_parameters', 'time_scope']),
+    filters: item.filters == null ? null : conditionTree(item.filters, item.relation),
+    operand_filters: list(item.custom_filters).map(filter => ({ index: filter.index, tree: conditionTree(filter, filter.relation) })),
+    operands: list(formula.formulationDeps).map((dependency, index) => ({ index,
+      ...pick(obj(dependency), ['event', 'quota', 'property', 'date', 'timeRange', 'time_scope', 'quotaTimeRange']),
+    })),
+    operand_time_ranges: item.quota_time_ranges ?? item.quotaTimeRanges ?? null,
+  };
+}
+
+export const modelConfigKeys = [
+  'windowsGapValue', 'windowsGapUnit', 'percentConfig', 'userCrowds', 'retentionType',
+  'rtnRateOrNum', 'statType', 'retentionCollectSet', 'relationProps', 'relationPropSplit',
+  'unitNum', 'eventModel', 'taIdMeasureVo',
+];
+const timeKeys = ['recentDay', 'startTime', 'endTime', 'startToNow', 'startToYesterday',
+  'timeParticleSize', 'firstDayOfWeek', 'zoneOffset', 'timezone', 'completeIncompletePeriod', 'hideIncompleteData'];
+const interpretedViewKeys = new Set([...timeKeys, ...modelConfigKeys, 'filts', 'groupBy', 'group_by',
+  'relation', 'sqlViewParams', 'graphShape', 'projectId', 'displayQuotas', 'displayGroups']);
+
+export function reportContract(report, context = report.calculation_context || {}) {
+  const view = obj(valueOf(context.event_view));
+  const savedParameters = context.saved_parameters ?? view.sqlViewParams ?? null;
+  const parameters = list(context.sql_params ?? context.parameters).map(parameter => ({ ...parameter,
+    name: parameter.name ?? parameter.paramName,
+  }));
+  return {
+    model: report.report_model,
+    groups: context.group_by ?? view.groupBy ?? view.group_by ?? [],
+    time: semanticTime({ ...pick(view, timeKeys), ...obj(context.time_scope) }),
+    filters: conditionTree(context.filters ?? view.filts ?? [], context.relation ?? view.relation),
+    metrics: list(report.metric_occurrences).map(metricContract),
+    model_config: { ...pick(view, modelConfigKeys), ...obj(context.model_config) },
+    unparsed_model_config: Object.fromEntries(Object.entries(view).filter(([key]) => !interpretedViewKeys.has(key))),
+    model_events: list(valueOf(context.source_events)).map(event => pick(obj(event), ['eventName', 'type', 'filts', 'relation', 'analysis', 'analysisParams', 'quota', 'quotaEntities'])),
+    parameters,
+    saved_parameters: Array.isArray(savedParameters) ? savedParameters.map(parameter => Object.keys(obj(parameter)).length ? semanticTime(parameter) : parameter) : savedParameters,
+    sql_projections: report.definition_kind === 'SQL' ? list(obj(context.sql_semantic_facts).final_select?.fields) : [],
+  };
+}
+
+export function reportSignature(report, context = report.calculation_context || {}, includeSavedSettings = true) {
+  const contract = reportContract(report, context);
+  contract.time = semanticTime(contract.time);
+  if (!includeSavedSettings) contract.unparsed_model_config = {};
+  if (report.definition_kind === 'SQL') {
+    const definitions = contract.parameters.map(parameter => pick(parameter, ['name', 'paramType', 'type', 'options', 'selectorItems', 'required', 'useTimezone', 'use_timezone']));
+    return JSON.stringify(semanticValue({ sql: String(context.sql || '').replace(/\r\n?/g, '\n').trim(),
+      parameters: definitions, ...(includeSavedSettings ? { saved_parameters: contract.saved_parameters, time: contract.time } : {}) }));
+  }
+  return JSON.stringify(semanticValue(contract));
+}
+
+export function enrichMetricEvidence(metrics, context) {
+  const source = list(valueOf(context.source_events));
+  return metrics.map((metric, index) => {
+    const raw = obj(source[index]);
+    if (source.length !== metrics.length || raw.eventName !== metric.event_name || String(raw.analysis ?? '') !== String(metric.aggregation_code ?? '')) return metric;
+    return { ...metric,
+      ...(raw.analysisParams != null ? { aggregation_parameters: raw.analysisParams } : {}),
+      ...(raw.quotaEntities != null ? { entity_grain: raw.quotaEntities } : {}),
+      ...(raw.quotaTimeRanges != null ? { quota_time_ranges: raw.quotaTimeRanges } : {}),
+    };
+  });
+}
+
+const inline = value => `\`${JSON.stringify(value).replaceAll('`', '\\`').replaceAll('\n', ' ')}\``;
+
+export function renderExecutionContract(contract) {
+  const rows = [
+    '- These are saved definition settings. Execution overrides and inherited dashboard settings require live verification.',
+    '- Saved result grouping is listed below. SQL output grain and inherited settings require the asset explanation; referenced metadata is not a saved group.',
+    ...contract.groups.map((group, index) => `- Group ${index + 1}: ${inline(group)}`),
+    `- Saved population filter structure: ${inline(contract.filters)}`,
+    `- Saved time settings: ${Object.keys(contract.time).length ? inline(contract.time) : 'unknown; verify the current definition'}`,
+    '- State timing: a historical behavior window does not establish historical values for user or dimension tables. Source refresh times and historical reconstruction require source evidence.',
+    '- Empty/missing rows, null, empty arrays, false and zero are distinct. Do not fill missing rows with zero without coverage evidence.',
+  ];
+  if (contract.groups.some(group => /array/i.test(JSON.stringify(group)))) rows.push('- Array grouping can expand an entity into several rows. Distinct counts, ratios and percentiles are not additive across those rows.');
+  if (Object.keys(contract.model_config).length) rows.push(`- Model-specific settings: ${inline(contract.model_config)}`);
+  if (Object.keys(contract.unparsed_model_config).length) rows.push(`- Uninterpreted saved configuration; semantic support is unknown: ${inline(contract.unparsed_model_config)}`);
+  if ((contract.model_config.windowsGapValue != null || list(contract.model_config.userCrowds).length) && contract.model_events.length) rows.push(`- Model event roles and measures: ${inline(contract.model_events)}`);
+  if (contract.parameters.length) rows.push(`- Parameter definitions: ${inline(contract.parameters)}`);
+  if (contract.saved_parameters != null) rows.push(`- Saved parameter choices (separate from definitions and call-time overrides): ${inline(contract.saved_parameters)}`);
+  if (contract.sql_projections.length) {
+    const projections = contract.sql_projections.filter(field => !/\$\{/.test(field.expression || ''));
+    rows.push(`- SQL outer projection evidence: ${inline(projections)}`);
+    rows.push('- SQL lineage completeness: unknown for nested scopes, CTEs, wildcard fields and dynamic expressions. Output names alone do not prove source meanings; verify unresolved columns.');
+  }
+  for (const [index, metric] of contract.metrics.entries()) {
+    if (metric.operands.length || metric.operand_filters.length || metric.aggregation_parameters != null || metric.entity_grain != null) {
+      rows.push(`- Measure ${index + 1} calculation scopes: ${inline(metric)}`);
+    }
+  }
+  return `\n\n## Execution contract\n\n${rows.join('\n')}\n`;
+}
+
+export function validateSqlExplanations(summary, sql) {
+  const normalize = value => String(value).replace(/\s+/g, ' ').trim();
+  for (const key of ['calculation_rules', 'time_semantics', 'coverage_gaps']) {
+    if (summary[key] == null) continue;
+    if (!Array.isArray(summary[key])) throw new Error(`${key} must be an array.`);
+    for (const item of summary[key]) {
+      if (!item?.meaning || !item?.evidence || !normalize(sql).includes(normalize(item.evidence))) {
+        throw new Error(`${key} requires a meaning and exact SQL evidence from the current definition.`);
+      }
+    }
+  }
+  for (const field of list(summary.output_fields)) {
+    if (field.evidence != null && !normalize(sql).includes(normalize(field.evidence))) {
+      throw new Error('Output field lineage evidence must occur in the current SQL.');
+    }
+  }
+}
+
+export function renderSqlExplanations(summary) {
+  return ['calculation_rules', 'time_semantics', 'coverage_gaps'].flatMap(key => list(summary?.[key])
+    .map(item => `- ${key}: ${item.meaning} (evidence: ${inline(item.evidence)})`)).join('\n');
+}