ai-database 2.1.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/CHANGELOG.md +47 -1
  2. package/README.md +1063 -186
  3. package/dist/actions.d.ts +2 -2
  4. package/dist/actions.d.ts.map +1 -1
  5. package/dist/actions.js +1 -1
  6. package/dist/actions.js.map +1 -1
  7. package/dist/ai-promise-db.d.ts +52 -23
  8. package/dist/ai-promise-db.d.ts.map +1 -1
  9. package/dist/ai-promise-db.js +185 -164
  10. package/dist/ai-promise-db.js.map +1 -1
  11. package/dist/authorization.d.ts.map +1 -1
  12. package/dist/authorization.js +38 -30
  13. package/dist/authorization.js.map +1 -1
  14. package/dist/cascade-orchestrator.d.ts +404 -0
  15. package/dist/cascade-orchestrator.d.ts.map +1 -0
  16. package/dist/cascade-orchestrator.js +828 -0
  17. package/dist/cascade-orchestrator.js.map +1 -0
  18. package/dist/cascade-write-strategy.d.ts +584 -0
  19. package/dist/cascade-write-strategy.d.ts.map +1 -0
  20. package/dist/cascade-write-strategy.js +590 -0
  21. package/dist/cascade-write-strategy.js.map +1 -0
  22. package/dist/ch-adapter.d.ts +358 -0
  23. package/dist/ch-adapter.d.ts.map +1 -0
  24. package/dist/ch-adapter.js +929 -0
  25. package/dist/ch-adapter.js.map +1 -0
  26. package/dist/client/index.d.ts +42 -0
  27. package/dist/client/index.d.ts.map +1 -0
  28. package/dist/client/index.js +43 -0
  29. package/dist/client/index.js.map +1 -0
  30. package/dist/client.d.ts +266 -0
  31. package/dist/client.d.ts.map +1 -0
  32. package/dist/client.js +81 -0
  33. package/dist/client.js.map +1 -0
  34. package/dist/constants.d.ts +64 -1
  35. package/dist/constants.d.ts.map +1 -1
  36. package/dist/constants.js +52 -2
  37. package/dist/constants.js.map +1 -1
  38. package/dist/dataloader.d.ts +99 -0
  39. package/dist/dataloader.d.ts.map +1 -0
  40. package/dist/dataloader.js +225 -0
  41. package/dist/dataloader.js.map +1 -0
  42. package/dist/db-provider-port.d.ts +501 -0
  43. package/dist/db-provider-port.d.ts.map +1 -0
  44. package/dist/db-provider-port.js +113 -0
  45. package/dist/db-provider-port.js.map +1 -0
  46. package/dist/digital-objects-provider.d.ts +49 -0
  47. package/dist/digital-objects-provider.d.ts.map +1 -0
  48. package/dist/digital-objects-provider.js +55 -0
  49. package/dist/digital-objects-provider.js.map +1 -0
  50. package/dist/do-sqlite-adapter.d.ts +402 -0
  51. package/dist/do-sqlite-adapter.d.ts.map +1 -0
  52. package/dist/do-sqlite-adapter.js +745 -0
  53. package/dist/do-sqlite-adapter.js.map +1 -0
  54. package/dist/docs-rels/custom-types.d.ts +134 -0
  55. package/dist/docs-rels/custom-types.d.ts.map +1 -0
  56. package/dist/docs-rels/custom-types.js +70 -0
  57. package/dist/docs-rels/custom-types.js.map +1 -0
  58. package/dist/docs-rels/index.d.ts +16 -0
  59. package/dist/docs-rels/index.d.ts.map +1 -0
  60. package/dist/docs-rels/index.js +16 -0
  61. package/dist/docs-rels/index.js.map +1 -0
  62. package/dist/docs-rels/migrations/index.d.ts +30 -0
  63. package/dist/docs-rels/migrations/index.d.ts.map +1 -0
  64. package/dist/docs-rels/migrations/index.js +128 -0
  65. package/dist/docs-rels/migrations/index.js.map +1 -0
  66. package/dist/docs-rels/schema.d.ts +2961 -0
  67. package/dist/docs-rels/schema.d.ts.map +1 -0
  68. package/dist/docs-rels/schema.js +244 -0
  69. package/dist/docs-rels/schema.js.map +1 -0
  70. package/dist/durable-clickhouse.d.ts.map +1 -1
  71. package/dist/durable-clickhouse.js +16 -13
  72. package/dist/durable-clickhouse.js.map +1 -1
  73. package/dist/durable-promise.d.ts.map +1 -1
  74. package/dist/durable-promise.js +34 -15
  75. package/dist/durable-promise.js.map +1 -1
  76. package/dist/errors.d.ts +127 -0
  77. package/dist/errors.d.ts.map +1 -0
  78. package/dist/errors.js +210 -0
  79. package/dist/errors.js.map +1 -0
  80. package/dist/eventbridge.d.ts +117 -0
  81. package/dist/eventbridge.d.ts.map +1 -0
  82. package/dist/eventbridge.js +238 -0
  83. package/dist/eventbridge.js.map +1 -0
  84. package/dist/events.d.ts +2 -2
  85. package/dist/events.d.ts.map +1 -1
  86. package/dist/events.js +1 -1
  87. package/dist/events.js.map +1 -1
  88. package/dist/execution-queue.d.ts.map +1 -1
  89. package/dist/execution-queue.js +4 -5
  90. package/dist/execution-queue.js.map +1 -1
  91. package/dist/index.d.ts +37 -8
  92. package/dist/index.d.ts.map +1 -1
  93. package/dist/index.js +112 -6
  94. package/dist/index.js.map +1 -1
  95. package/dist/linguistic.d.ts +3 -108
  96. package/dist/linguistic.d.ts.map +1 -1
  97. package/dist/linguistic.js +3 -372
  98. package/dist/linguistic.js.map +1 -1
  99. package/dist/logger.d.ts +132 -0
  100. package/dist/logger.d.ts.map +1 -0
  101. package/dist/logger.js +137 -0
  102. package/dist/logger.js.map +1 -0
  103. package/dist/memory-provider.d.ts +129 -0
  104. package/dist/memory-provider.d.ts.map +1 -1
  105. package/dist/memory-provider.js +592 -257
  106. package/dist/memory-provider.js.map +1 -1
  107. package/dist/pg-adapter.d.ts +424 -0
  108. package/dist/pg-adapter.d.ts.map +1 -0
  109. package/dist/pg-adapter.js +921 -0
  110. package/dist/pg-adapter.js.map +1 -0
  111. package/dist/pipelines-iceberg-emitter.d.ts +327 -0
  112. package/dist/pipelines-iceberg-emitter.d.ts.map +1 -0
  113. package/dist/pipelines-iceberg-emitter.js +351 -0
  114. package/dist/pipelines-iceberg-emitter.js.map +1 -0
  115. package/dist/provider-capabilities.d.ts +146 -0
  116. package/dist/provider-capabilities.d.ts.map +1 -0
  117. package/dist/provider-capabilities.js +214 -0
  118. package/dist/provider-capabilities.js.map +1 -0
  119. package/dist/rdb-provider-adapter.d.ts +195 -0
  120. package/dist/rdb-provider-adapter.d.ts.map +1 -0
  121. package/dist/rdb-provider-adapter.js +291 -0
  122. package/dist/rdb-provider-adapter.js.map +1 -0
  123. package/dist/schema/cascade.d.ts +49 -10
  124. package/dist/schema/cascade.d.ts.map +1 -1
  125. package/dist/schema/cascade.js +491 -273
  126. package/dist/schema/cascade.js.map +1 -1
  127. package/dist/schema/definition-caches.d.ts +24 -0
  128. package/dist/schema/definition-caches.d.ts.map +1 -0
  129. package/dist/schema/definition-caches.js +26 -0
  130. package/dist/schema/definition-caches.js.map +1 -0
  131. package/dist/schema/dependency-graph.d.ts +45 -0
  132. package/dist/schema/dependency-graph.d.ts.map +1 -0
  133. package/dist/schema/dependency-graph.js +47 -0
  134. package/dist/schema/dependency-graph.js.map +1 -0
  135. package/dist/schema/diff.d.ts +103 -0
  136. package/dist/schema/diff.d.ts.map +1 -0
  137. package/dist/schema/diff.js +329 -0
  138. package/dist/schema/diff.js.map +1 -0
  139. package/dist/schema/entity-operations.d.ts +99 -0
  140. package/dist/schema/entity-operations.d.ts.map +1 -0
  141. package/dist/schema/entity-operations.js +818 -0
  142. package/dist/schema/entity-operations.js.map +1 -0
  143. package/dist/schema/generation-context.d.ts +202 -0
  144. package/dist/schema/generation-context.d.ts.map +1 -0
  145. package/dist/schema/generation-context.js +393 -0
  146. package/dist/schema/generation-context.js.map +1 -0
  147. package/dist/schema/index.d.ts +32 -34
  148. package/dist/schema/index.d.ts.map +1 -1
  149. package/dist/schema/index.js +462 -519
  150. package/dist/schema/index.js.map +1 -1
  151. package/dist/schema/migration.d.ts +205 -0
  152. package/dist/schema/migration.d.ts.map +1 -0
  153. package/dist/schema/migration.js +327 -0
  154. package/dist/schema/migration.js.map +1 -0
  155. package/dist/schema/nl-query-generator.d.ts +68 -0
  156. package/dist/schema/nl-query-generator.d.ts.map +1 -0
  157. package/dist/schema/nl-query-generator.js +362 -0
  158. package/dist/schema/nl-query-generator.js.map +1 -0
  159. package/dist/schema/nl-query.d.ts +65 -0
  160. package/dist/schema/nl-query.d.ts.map +1 -0
  161. package/dist/schema/nl-query.js +178 -0
  162. package/dist/schema/nl-query.js.map +1 -0
  163. package/dist/schema/parse.d.ts.map +1 -1
  164. package/dist/schema/parse.js +152 -89
  165. package/dist/schema/parse.js.map +1 -1
  166. package/dist/schema/provider.d.ts +38 -0
  167. package/dist/schema/provider.d.ts.map +1 -1
  168. package/dist/schema/provider.js +15 -7
  169. package/dist/schema/provider.js.map +1 -1
  170. package/dist/schema/resolve.d.ts +46 -5
  171. package/dist/schema/resolve.d.ts.map +1 -1
  172. package/dist/schema/resolve.js +334 -117
  173. package/dist/schema/resolve.js.map +1 -1
  174. package/dist/schema/search-utils.d.ts +76 -0
  175. package/dist/schema/search-utils.d.ts.map +1 -0
  176. package/dist/schema/search-utils.js +86 -0
  177. package/dist/schema/search-utils.js.map +1 -0
  178. package/dist/schema/seed.d.ts +53 -0
  179. package/dist/schema/seed.d.ts.map +1 -0
  180. package/dist/schema/seed.js +94 -0
  181. package/dist/schema/seed.js.map +1 -0
  182. package/dist/schema/semantic.d.ts +11 -0
  183. package/dist/schema/semantic.d.ts.map +1 -1
  184. package/dist/schema/semantic.js +262 -68
  185. package/dist/schema/semantic.js.map +1 -1
  186. package/dist/schema/sub-apis.d.ts +52 -0
  187. package/dist/schema/sub-apis.d.ts.map +1 -0
  188. package/dist/schema/sub-apis.js +216 -0
  189. package/dist/schema/sub-apis.js.map +1 -0
  190. package/dist/schema/system-entities.d.ts +42 -0
  191. package/dist/schema/system-entities.d.ts.map +1 -0
  192. package/dist/schema/system-entities.js +101 -0
  193. package/dist/schema/system-entities.js.map +1 -0
  194. package/dist/schema/types.d.ts +91 -9
  195. package/dist/schema/types.d.ts.map +1 -1
  196. package/dist/schema/union-fallback.d.ts +219 -0
  197. package/dist/schema/union-fallback.d.ts.map +1 -0
  198. package/dist/schema/union-fallback.js +331 -0
  199. package/dist/schema/union-fallback.js.map +1 -0
  200. package/dist/schema/value-generators/ai.d.ts +54 -0
  201. package/dist/schema/value-generators/ai.d.ts.map +1 -0
  202. package/dist/schema/value-generators/ai.js +136 -0
  203. package/dist/schema/value-generators/ai.js.map +1 -0
  204. package/dist/schema/value-generators/index.d.ts +126 -0
  205. package/dist/schema/value-generators/index.d.ts.map +1 -0
  206. package/dist/schema/value-generators/index.js +219 -0
  207. package/dist/schema/value-generators/index.js.map +1 -0
  208. package/dist/schema/value-generators/placeholder.d.ts +52 -0
  209. package/dist/schema/value-generators/placeholder.d.ts.map +1 -0
  210. package/dist/schema/value-generators/placeholder.js +328 -0
  211. package/dist/schema/value-generators/placeholder.js.map +1 -0
  212. package/dist/schema/value-generators/types.d.ts +116 -0
  213. package/dist/schema/value-generators/types.d.ts.map +1 -0
  214. package/dist/schema/value-generators/types.js +11 -0
  215. package/dist/schema/value-generators/types.js.map +1 -0
  216. package/dist/schema/verb-derivation.d.ts +167 -0
  217. package/dist/schema/verb-derivation.d.ts.map +1 -0
  218. package/dist/schema/verb-derivation.js +281 -0
  219. package/dist/schema/verb-derivation.js.map +1 -0
  220. package/dist/schema/version.d.ts +111 -0
  221. package/dist/schema/version.d.ts.map +1 -0
  222. package/dist/schema/version.js +190 -0
  223. package/dist/schema/version.js.map +1 -0
  224. package/dist/schema.d.ts +1095 -23
  225. package/dist/schema.d.ts.map +1 -1
  226. package/dist/schema.js +2854 -38
  227. package/dist/schema.js.map +1 -1
  228. package/dist/semantic-vectors.d.ts +39 -0
  229. package/dist/semantic-vectors.d.ts.map +1 -0
  230. package/dist/semantic-vectors.js +334 -0
  231. package/dist/semantic-vectors.js.map +1 -0
  232. package/dist/semantic.d.ts +29 -1
  233. package/dist/semantic.d.ts.map +1 -1
  234. package/dist/semantic.js +26 -16
  235. package/dist/semantic.js.map +1 -1
  236. package/dist/telemetry.d.ts +128 -0
  237. package/dist/telemetry.d.ts.map +1 -0
  238. package/dist/telemetry.js +305 -0
  239. package/dist/telemetry.js.map +1 -0
  240. package/dist/tests.d.ts.map +1 -1
  241. package/dist/tests.js +30 -22
  242. package/dist/tests.js.map +1 -1
  243. package/dist/type-guards.d.ts +212 -0
  244. package/dist/type-guards.d.ts.map +1 -0
  245. package/dist/type-guards.js +318 -0
  246. package/dist/type-guards.js.map +1 -0
  247. package/dist/types.d.ts +33 -245
  248. package/dist/types.d.ts.map +1 -1
  249. package/dist/types.js +62 -72
  250. package/dist/types.js.map +1 -1
  251. package/dist/validation.d.ts +165 -0
  252. package/dist/validation.d.ts.map +1 -0
  253. package/dist/validation.js +639 -0
  254. package/dist/validation.js.map +1 -0
  255. package/dist/worker/db-provider.d.ts +168 -0
  256. package/dist/worker/db-provider.d.ts.map +1 -0
  257. package/dist/worker/db-provider.js +277 -0
  258. package/dist/worker/db-provider.js.map +1 -0
  259. package/dist/worker/index.d.ts +35 -0
  260. package/dist/worker/index.d.ts.map +1 -0
  261. package/dist/worker/index.js +37 -0
  262. package/dist/worker/index.js.map +1 -0
  263. package/dist/worker.d.ts +779 -0
  264. package/dist/worker.d.ts.map +1 -0
  265. package/dist/worker.js +2786 -0
  266. package/dist/worker.js.map +1 -0
  267. package/package.json +38 -8
  268. package/src/docs-rels/migrations/0001-init.sql +125 -0
@@ -8,6 +8,7 @@
8
8
  */
9
9
  import { hasSemanticSearch } from './provider.js';
10
10
  import { isPrimitiveType } from './parse.js';
11
+ import { findBestMatchAcrossTypes } from './search-utils.js';
11
12
  // =============================================================================
12
13
  // Context Resolution - Template Variables and $instructions
13
14
  // =============================================================================
@@ -104,63 +105,132 @@ export async function resolveInstructions(instructions, entity, typeName, schema
104
105
  return resolved;
105
106
  }
106
107
  /**
107
- * Pre-fetch context dependencies declared in $context
108
+ * Find the field in the entity that references a given type
109
+ */
110
+ function findFieldByRelatedType(entityDef, relatedType) {
111
+ for (const [fieldName, field] of entityDef.fields) {
112
+ if (field.isRelation && field.relatedType === relatedType) {
113
+ return { fieldName, field };
114
+ }
115
+ }
116
+ return undefined;
117
+ }
118
+ /**
119
+ * Pre-fetch context dependencies declared in $context using dotted paths
108
120
  *
109
121
  * The $context field declares explicit context dependencies that should
110
- * be pre-fetched before generating AI fields. This ensures all referenced
111
- * entities are available for template variable resolution.
122
+ * be pre-fetched before generating AI fields. Supports dotted paths like
123
+ * 'Startup.icp.industry' for deeply nested context fetching.
112
124
  *
113
- * @param contextDeps - Array of context dependency types (e.g., ['Startup', 'ICP'])
125
+ * Context paths can be:
126
+ * - Simple type names: 'Startup' - finds field that references Startup type
127
+ * - Dotted paths: 'Startup.icp.industry' - traverses relationships
128
+ *
129
+ * @param contextDeps - Array of context dependency paths (e.g., ['Startup', 'Startup.icp.industry'])
114
130
  * @param entity - The current entity data
115
131
  * @param typeName - The current entity type name
116
132
  * @param schema - The parsed schema
117
133
  * @param provider - The database provider for fetching related entities
118
- * @returns Map of context name to fetched entity data
134
+ * @returns Map of context path to fetched entity data
119
135
  */
120
- export async function prefetchContext(contextDeps, entity, typeName, schema, provider) {
136
+ export async function prefetchContextPaths(contextDeps, entity, typeName, schema, provider) {
121
137
  const contextData = new Map();
122
138
  const currentEntity = schema.entities.get(typeName);
123
139
  if (!currentEntity)
124
140
  return contextData;
125
- for (const dep of contextDeps) {
126
- // Convert to camelCase for field lookup (e.g., "Startup" -> "startup")
127
- const fieldName = dep.charAt(0).toLowerCase() + dep.slice(1);
128
- // Check if we have a field that references this type
129
- const field = currentEntity.fields.get(fieldName);
130
- if (field?.isRelation && field.relatedType) {
131
- const entityId = entity[fieldName];
132
- if (typeof entityId === 'string') {
133
- const fetched = await provider.get(field.relatedType, entityId);
134
- if (fetched) {
135
- contextData.set(fieldName, fetched);
136
- // Also store nested relationships
137
- const relatedEntity = schema.entities.get(field.relatedType);
138
- if (relatedEntity) {
139
- for (const [nestedFieldName, nestedField] of relatedEntity.fields) {
140
- if (nestedField.isRelation && nestedField.relatedType) {
141
- const nestedId = fetched[nestedFieldName];
142
- if (typeof nestedId === 'string') {
143
- const nestedFetched = await provider.get(nestedField.relatedType, nestedId);
144
- if (nestedFetched) {
145
- contextData.set(`${fieldName}.${nestedFieldName}`, nestedFetched);
146
- }
147
- }
148
- }
149
- }
150
- }
141
+ // Track already fetched entities to avoid duplicate fetches
142
+ // Key: "TypeName:entityId", Value: fetched entity data
143
+ const fetchCache = new Map();
144
+ // Deduplicate context deps
145
+ const uniqueDeps = [...new Set(contextDeps)];
146
+ for (const dep of uniqueDeps) {
147
+ // Parse the path - could be "Startup" or "Startup.icp.industry"
148
+ const pathParts = dep.split('.');
149
+ // Start from the current entity and traverse the path
150
+ let currentData = entity;
151
+ let currentEntityDef = currentEntity;
152
+ let currentPath = '';
153
+ for (let i = 0; i < pathParts.length; i++) {
154
+ const part = pathParts[i];
155
+ // Try two approaches:
156
+ // 1. Convert PascalCase type name to camelCase field name
157
+ // 2. Find field by related type
158
+ const camelCaseFieldName = part.charAt(0).toLowerCase() + part.slice(1);
159
+ let fieldInfo;
160
+ // First try exact field name match
161
+ const directField = currentEntityDef.fields.get(camelCaseFieldName);
162
+ if (directField?.isRelation && directField.relatedType) {
163
+ fieldInfo = { fieldName: camelCaseFieldName, field: directField };
164
+ }
165
+ // If no direct match, try to find field by related type (e.g., 'Related' -> find field that points to Related)
166
+ if (!fieldInfo) {
167
+ fieldInfo = findFieldByRelatedType(currentEntityDef, part);
168
+ }
169
+ if (!fieldInfo) {
170
+ // Path invalid - no field found
171
+ break;
172
+ }
173
+ const { fieldName, field } = fieldInfo;
174
+ currentPath = currentPath ? `${currentPath}.${fieldName}` : fieldName;
175
+ // Get the entity ID from current data
176
+ const entityId = currentData[fieldName];
177
+ if (typeof entityId !== 'string') {
178
+ // Path invalid - no entity ID
179
+ break;
180
+ }
181
+ // Check cache first
182
+ const cacheKey = `${field.relatedType}:${entityId}`;
183
+ let fetched = fetchCache.get(cacheKey);
184
+ if (!fetched) {
185
+ // Fetch the related entity
186
+ const fetchedResult = await provider.get(field.relatedType, entityId);
187
+ if (fetchedResult) {
188
+ fetched = fetchedResult;
189
+ fetchCache.set(cacheKey, fetched);
151
190
  }
152
191
  }
192
+ if (!fetched) {
193
+ // Path invalid - entity not found
194
+ break;
195
+ }
196
+ // Store the result with the actual field name path
197
+ contextData.set(currentPath, fetched);
198
+ // Update for next iteration
199
+ currentData = fetched;
200
+ const nextEntityDef = schema.entities.get(field.relatedType);
201
+ if (!nextEntityDef) {
202
+ // Path invalid - entity type not in schema
203
+ break;
204
+ }
205
+ currentEntityDef = nextEntityDef;
153
206
  }
154
207
  }
155
208
  return contextData;
156
209
  }
157
210
  /**
158
- * Check if a field type is a prompt (contains spaces, indicating AI generation)
211
+ * Pre-fetch context dependencies declared in $context
212
+ *
213
+ * The $context field declares explicit context dependencies that should
214
+ * be pre-fetched before generating AI fields. This ensures all referenced
215
+ * entities are available for template variable resolution.
216
+ *
217
+ * @param contextDeps - Array of context dependency types (e.g., ['Startup', 'ICP'])
218
+ * @param entity - The current entity data
219
+ * @param typeName - The current entity type name
220
+ * @param schema - The parsed schema
221
+ * @param provider - The database provider for fetching related entities
222
+ * @returns Map of context name to fetched entity data
223
+ */
224
+ export async function prefetchContext(contextDeps, entity, typeName, schema, provider) {
225
+ // Use the new path-based implementation which handles both simple and dotted paths
226
+ return prefetchContextPaths(contextDeps, entity, typeName, schema, provider);
227
+ }
228
+ /**
229
+ * Check if a field type is a prompt (contains spaces, slashes, or question marks indicating AI generation).
230
+ * Examples: 'Describe the product', 'low/medium/high', 'What is the severity?'
159
231
  */
160
232
  export function isPromptField(field) {
161
- // Fields with spaces in their type are prompts for AI generation
162
- return field.type.includes(' ') ||
163
- (field.type === 'string' && !field.isRelation && !isPrimitiveType(field.type));
233
+ return field.type.includes(' ') || field.type.includes('/') || field.type.includes('?');
164
234
  }
165
235
  // =============================================================================
166
236
  // Nested Pending Resolution
@@ -184,7 +254,7 @@ export async function resolveNestedPending(data, entity, schema, provider) {
184
254
  if (relatedEntity) {
185
255
  const resolvedNested = await resolveNestedPending(pending.data, relatedEntity, schema, provider);
186
256
  const created = await provider.create(pending.type, undefined, resolvedNested);
187
- resolved[fieldName] = created.$id;
257
+ resolved[fieldName] = created['$id'];
188
258
  }
189
259
  }
190
260
  }
@@ -199,7 +269,7 @@ export async function resolveNestedPending(data, entity, schema, provider) {
199
269
  * For exact matches (-> and <-), creates new entities.
200
270
  * For fuzzy matches (~> and <~), searches for existing entities first.
201
271
  */
202
- export async function resolveReferenceSpec(spec, contextData, schema, provider, generateContextAwareValue) {
272
+ export async function resolveReferenceSpec(spec, contextData, schema, provider, generateContextAwareValue, generateEntityWithAI) {
203
273
  const targetEntity = schema.entities.get(spec.type);
204
274
  if (!targetEntity) {
205
275
  throw new Error(`Unknown target type: ${spec.type}`);
@@ -208,66 +278,123 @@ export async function resolveReferenceSpec(spec, contextData, schema, provider,
208
278
  // For fuzzy references, try to find an existing entity first
209
279
  if (hasSemanticSearch(provider)) {
210
280
  const searchQuery = spec.generatedText || spec.prompt || spec.field;
211
- const matches = await provider.semanticSearch(spec.type, searchQuery, {
212
- minScore: 0.5,
213
- limit: 1,
281
+ // Use threshold from spec, or default to 0.75
282
+ const threshold = spec.threshold ?? 0.75;
283
+ // Determine which types to search - all union types or just the primary type
284
+ const typesToSearch = spec.unionTypes && spec.unionTypes.length > 0 ? spec.unionTypes : [spec.type];
285
+ // Search all types and find the best match using shared utility
286
+ const bestMatchResult = await findBestMatchAcrossTypes(typesToSearch, searchQuery, {
287
+ threshold,
288
+ limit: 5,
289
+ schema,
290
+ provider,
214
291
  });
215
- const firstMatch = matches[0];
216
- if (firstMatch) {
217
- return firstMatch.$id;
292
+ if (bestMatchResult) {
293
+ // Update the matched entity with metadata
294
+ await provider.update(bestMatchResult.type, bestMatchResult.match.$id, {
295
+ $matchedType: bestMatchResult.type,
296
+ $similarity: bestMatchResult.match.$score,
297
+ });
298
+ // Also set metadata on contextData for edge creation later
299
+ contextData[`${spec.field}$matchedType`] = bestMatchResult.type;
300
+ contextData[`${spec.field}$score`] = bestMatchResult.match.$score;
301
+ return bestMatchResult.match.$id;
218
302
  }
219
303
  }
220
304
  // If no match found for fuzzy, fall through to create
221
305
  }
222
306
  // Create a new entity
223
- const generatedData = {};
224
- // Build context for generation
307
+ let generatedData = {};
225
308
  const hint = spec.generatedText || spec.prompt || spec.field;
226
- const parentContextFields = [];
227
- for (const [key, value] of Object.entries(contextData)) {
228
- if (!key.startsWith('$') && !key.startsWith('_') && typeof value === 'string' && value) {
229
- parentContextFields.push(`${key}: ${value}`);
309
+ const parentType = contextData['$type'];
310
+ const parentId = contextData['$id'];
311
+ // Try AI generation first if available
312
+ if (generateEntityWithAI && parentType) {
313
+ try {
314
+ const aiData = await generateEntityWithAI(spec.type, hint, {
315
+ parent: parentType,
316
+ parentData: contextData,
317
+ ...(parentId !== undefined ? { parentId } : {}),
318
+ }, schema);
319
+ if (aiData && Object.keys(aiData).length > 0) {
320
+ // Resolve any nested pending relations (from generateEntity)
321
+ generatedData = await resolveNestedPending(aiData, targetEntity, schema, provider);
322
+ }
323
+ }
324
+ catch {
325
+ // AI generation failed - fall through to placeholder generation
230
326
  }
231
327
  }
232
- const fullContext = [hint, ...parentContextFields].filter(Boolean).join(' | ');
233
- // Generate default values for the target entity's fields
234
- for (const [fieldName, field] of targetEntity.fields) {
235
- if (!field.isRelation && !field.isOptional) {
236
- if (field.type === 'string') {
237
- generatedData[fieldName] = generateContextAwareValue(fieldName, spec.type, fullContext, hint, contextData);
238
- }
239
- else if (field.type === 'number') {
240
- generatedData[fieldName] = 0;
241
- }
242
- else if (field.type === 'boolean') {
243
- generatedData[fieldName] = false;
328
+ // If AI generation didn't produce data, fall back to placeholder generation
329
+ if (Object.keys(generatedData).length === 0) {
330
+ // Build context for generation
331
+ const parentContextFields = [];
332
+ for (const [key, value] of Object.entries(contextData)) {
333
+ if (!key.startsWith('$') && !key.startsWith('_') && typeof value === 'string' && value) {
334
+ parentContextFields.push(`${key}: ${value}`);
244
335
  }
245
336
  }
246
- else if (field.isRelation && field.operator === '->' && !field.isArray && !field.isOptional) {
247
- // Recursively resolve nested forward exact relations
248
- const nestedSpec = {
249
- field: fieldName,
250
- operator: '->',
251
- type: field.relatedType,
252
- matchMode: 'exact',
253
- resolved: false,
254
- prompt: field.prompt,
255
- };
256
- const nestedId = await resolveReferenceSpec(nestedSpec, generatedData, schema, provider, generateContextAwareValue);
257
- if (nestedId) {
258
- generatedData[fieldName] = nestedId;
337
+ // Include source entity's $instructions (parent context) and target entity's $instructions
338
+ const sourceInstructions = spec.sourceInstructions;
339
+ const targetInstructions = targetEntity.schema?.['$instructions'];
340
+ const contextParts = [];
341
+ // Source instructions first (parent context propagation)
342
+ if (sourceInstructions) {
343
+ contextParts.push(sourceInstructions);
344
+ }
345
+ // Then target instructions (child entity's own context)
346
+ if (targetInstructions) {
347
+ contextParts.push(targetInstructions);
348
+ }
349
+ if (hint) {
350
+ contextParts.push(hint);
351
+ }
352
+ contextParts.push(...parentContextFields);
353
+ const fullContext = contextParts.filter(Boolean).join(' | ');
354
+ // Generate default values for the target entity's fields
355
+ for (const [fieldName, field] of targetEntity.fields) {
356
+ if (!field.isRelation && !field.isOptional) {
357
+ const isPrompt = isPromptField(field);
358
+ if (field.type === 'string' || isPrompt) {
359
+ // For prompt fields, use the type as additional context for generation
360
+ const fieldHint = isPrompt ? field.type : hint;
361
+ generatedData[fieldName] = generateContextAwareValue(fieldName, spec.type, fullContext, fieldHint, contextData);
362
+ }
363
+ else if (field.type === 'number') {
364
+ generatedData[fieldName] = 0;
365
+ }
366
+ else if (field.type === 'boolean') {
367
+ generatedData[fieldName] = false;
368
+ }
369
+ }
370
+ else if (field.isRelation &&
371
+ field.operator === '->' &&
372
+ !field.isArray &&
373
+ !field.isOptional) {
374
+ // Recursively resolve nested forward exact relations
375
+ const nestedSpec = {
376
+ field: fieldName,
377
+ operator: '->',
378
+ type: field.relatedType,
379
+ matchMode: 'exact',
380
+ resolved: false,
381
+ ...(field.prompt !== undefined ? { prompt: field.prompt } : {}),
382
+ };
383
+ const nestedId = await resolveReferenceSpec(nestedSpec, generatedData, schema, provider, generateContextAwareValue, generateEntityWithAI);
384
+ if (nestedId) {
385
+ generatedData[fieldName] = nestedId;
386
+ }
259
387
  }
260
388
  }
261
389
  }
262
390
  // Use parent ID if available, otherwise use a descriptive string
263
- const parentId = contextData.$id;
264
391
  const created = await provider.create(spec.type, undefined, {
265
392
  ...generatedData,
266
393
  $generated: true,
267
394
  $generatedBy: parentId || (spec.matchMode === 'fuzzy' ? 'fuzzy-resolution' : 'reference-resolution'),
268
395
  $sourceField: spec.field,
269
396
  });
270
- return created.$id;
397
+ return created['$id'];
271
398
  }
272
399
  // =============================================================================
273
400
  // Entity Hydration
@@ -279,14 +406,31 @@ export async function resolveReferenceSpec(spec, contextData, schema, provider,
279
406
  * of the related type that have a reference pointing TO this entity.
280
407
  * This enables reverse lookups like "get all comments for a post".
281
408
  *
282
- * Backward reference resolution:
283
- * - Single backward ref with stored ID: resolve directly (e.g., member.team = teamId -> get Team by ID)
409
+ * ## Backward Exact (<-) vs Backward Fuzzy (<~) Resolution
410
+ *
411
+ * This function handles both backward exact and backward fuzzy resolution,
412
+ * but the resolution strategies differ:
413
+ *
414
+ * **Backward Exact (`<-`)**: Uses foreign key lookup (exact ID match)
415
+ * - Queries `provider.list(type, { where: { backrefField: id } })`
416
+ * - Returns all entities that explicitly reference the current entity
417
+ * - Example: `Blog.posts: ['<-Post']` finds Posts where `post.blog === blog.$id`
418
+ *
419
+ * **Backward Fuzzy (`<~`)**: Uses pre-resolved IDs from semantic search
420
+ * - During creation, `resolveBackwardFuzzy()` stores matched IDs
421
+ * - Hydration retrieves entities using those stored IDs
422
+ * - Example: `ICP.as: '<~Occupation'` was resolved via semantic search at creation time
423
+ *
424
+ * ## Resolution Cases
425
+ *
426
+ * - Single backward ref with stored ID: resolve directly (e.g., member.team = teamId)
284
427
  * - Single backward ref without stored ID: find related entity that points to us via relations
285
- * - Array backward ref: find all entities of related type where their forward ref points to us
428
+ * - Array backward ref with stored IDs: resolve each stored ID (backward fuzzy case)
429
+ * - Array backward ref without stored IDs: query via backref lookup (backward exact case)
286
430
  */
287
431
  export function hydrateEntity(data, entity, schema, resolveProvider) {
288
432
  const hydrated = { ...data };
289
- const id = (data.$id || data.id);
433
+ const id = (data['$id'] || data['id']);
290
434
  const typeName = entity.name;
291
435
  // Add lazy getters for relations
292
436
  for (const [fieldName, field] of entity.fields) {
@@ -301,14 +445,40 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
301
445
  // - Can be awaited to get the related entity (thenable)
302
446
  if (!isBackward && !field.isArray && data[fieldName]) {
303
447
  const storedId = data[fieldName];
448
+ // For union types, we need to check all possible types
449
+ const typesToSearch = field.unionTypes && field.unionTypes.length > 0 ? field.unionTypes : [field.relatedType];
450
+ // Check if we have a stored matchedType from the creation
451
+ const storedMatchedType = data[`${fieldName}$matchedType`];
304
452
  const thenableProxy = new Proxy({}, {
305
453
  get(target, prop) {
306
454
  if (prop === 'then') {
307
455
  return (resolve, reject) => {
308
456
  return (async () => {
309
457
  const provider = await resolveProvider();
310
- const result = await provider.get(field.relatedType, storedId);
311
- return result ? hydrateEntity(result, relatedEntity, schema, resolveProvider) : null;
458
+ // If we have a stored matched type, use it directly
459
+ if (storedMatchedType && typesToSearch.includes(storedMatchedType)) {
460
+ const result = await provider.get(storedMatchedType, storedId);
461
+ if (result) {
462
+ const matchedEntity = schema.entities.get(storedMatchedType);
463
+ if (matchedEntity) {
464
+ const hydrated = hydrateEntity(result, matchedEntity, schema, resolveProvider);
465
+ return { ...hydrated, $matchedType: storedMatchedType };
466
+ }
467
+ }
468
+ }
469
+ // For union types, try each type until we find the entity
470
+ for (const searchType of typesToSearch) {
471
+ const searchEntity = schema.entities.get(searchType);
472
+ if (!searchEntity)
473
+ continue;
474
+ const result = await provider.get(searchType, storedId);
475
+ if (result) {
476
+ const hydrated = hydrateEntity(result, searchEntity, schema, resolveProvider);
477
+ // Include $matchedType to indicate which union type matched
478
+ return { ...hydrated, $matchedType: searchType };
479
+ }
480
+ }
481
+ return null;
312
482
  })().then(resolve, reject);
313
483
  };
314
484
  }
@@ -322,8 +492,14 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
322
492
  return (regex) => storedId.match(regex);
323
493
  }
324
494
  if (prop === '$type') {
325
- return field.relatedType;
495
+ return storedMatchedType || field.relatedType;
326
496
  }
497
+ // Return the stored ID for $id property access (useful for equality checks)
498
+ if (prop === '$id') {
499
+ return storedId;
500
+ }
501
+ // For any other property access, return undefined
502
+ // Note: Use problem.task.$id or String(problem.task) for equality checks
327
503
  return undefined;
328
504
  },
329
505
  });
@@ -336,6 +512,8 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
336
512
  // Note: we create the proxy even for empty arrays so they have $type for batch loading detection
337
513
  if (!isBackward && field.isArray && Array.isArray(data[fieldName])) {
338
514
  const storedIds = data[fieldName];
515
+ // For union types, we need to check all possible types
516
+ const typesToSearch = field.unionTypes && field.unionTypes.length > 0 ? field.unionTypes : [field.relatedType];
339
517
  // Create a proxy that wraps the array but adds thenable behavior
340
518
  const thenableArray = new Proxy(storedIds, {
341
519
  get(target, prop) {
@@ -343,8 +521,23 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
343
521
  return (resolve, reject) => {
344
522
  return (async () => {
345
523
  const provider = await resolveProvider();
346
- const results = await Promise.all(storedIds.map(targetId => provider.get(field.relatedType, targetId)));
347
- return results.filter(r => r !== null).map((r) => hydrateEntity(r, relatedEntity, schema, resolveProvider));
524
+ const results = [];
525
+ for (const targetId of storedIds) {
526
+ // For union types, try each type until we find the entity
527
+ for (const searchType of typesToSearch) {
528
+ const searchEntity = schema.entities.get(searchType);
529
+ if (!searchEntity)
530
+ continue;
531
+ const result = await provider.get(searchType, targetId);
532
+ if (result) {
533
+ const hydrated = hydrateEntity(result, searchEntity, schema, resolveProvider);
534
+ // Include $matchedType to indicate which union type matched
535
+ results.push({ ...hydrated, $matchedType: searchType });
536
+ break;
537
+ }
538
+ }
539
+ }
540
+ return results;
348
541
  })().then(resolve, reject);
349
542
  };
350
543
  }
@@ -381,9 +574,7 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
381
574
  if (storedId) {
382
575
  // Has stored ID - directly fetch the related entity
383
576
  const result = await provider.get(field.relatedType, storedId);
384
- return result
385
- ? hydrateEntity(result, relatedEntity, schema, resolveProvider)
386
- : null;
577
+ return result ? hydrateEntity(result, relatedEntity, schema, resolveProvider) : null;
387
578
  }
388
579
  // No stored ID - find via inverse relation lookup
389
580
  // Find entities of relatedType that have this entity in their relations
@@ -396,9 +587,9 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
396
587
  // Check if any entity of relatedType has this entity in that relation
397
588
  const allRelated = await provider.list(field.relatedType);
398
589
  for (const candidate of allRelated) {
399
- const candidateId = (candidate.$id || candidate.id);
590
+ const candidateId = (candidate['$id'] || candidate['id']);
400
591
  const related = await provider.related(field.relatedType, candidateId, relFieldName);
401
- if (related.some(r => (r.$id || r.id) === id)) {
592
+ if (related.some((r) => (r['$id'] || r['id']) === id)) {
402
593
  return hydrateEntity(candidate, relatedEntity, schema, resolveProvider);
403
594
  }
404
595
  }
@@ -412,40 +603,68 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
412
603
  const provider = await resolveProvider();
413
604
  if (isBackward) {
414
605
  // Case 2: Array backward ref
606
+ // For union types, we need to check all possible types
607
+ const typesToSearch = field.unionTypes && field.unionTypes.length > 0
608
+ ? field.unionTypes
609
+ : [field.relatedType];
415
610
  // Check if we have stored IDs (e.g., from backward fuzzy resolution)
416
611
  const storedIds = data[fieldName];
417
612
  if (Array.isArray(storedIds) && storedIds.length > 0) {
418
613
  // Use stored IDs directly - this handles backward fuzzy (<~) array fields
419
- const results = await Promise.all(storedIds.map(targetId => provider.get(field.relatedType, targetId)));
420
- return Promise.all(results.filter(r => r !== null).map((r) => hydrateEntity(r, relatedEntity, schema, resolveProvider)));
614
+ // For union types, try each type until we find the entity
615
+ const hydrated = [];
616
+ for (const targetId of storedIds) {
617
+ for (const searchType of typesToSearch) {
618
+ const searchEntity = schema.entities.get(searchType);
619
+ if (!searchEntity)
620
+ continue;
621
+ const result = await provider.get(searchType, targetId);
622
+ if (result) {
623
+ const hydratedEntity = hydrateEntity(result, searchEntity, schema, resolveProvider);
624
+ hydrated.push({ ...hydratedEntity, $matchedType: searchType });
625
+ break;
626
+ }
627
+ }
628
+ }
629
+ return hydrated;
421
630
  }
422
- // No stored IDs - use backref lookup
631
+ // No stored IDs - use backref lookup across all union types
423
632
  // e.g., Blog.posts: ['<-Post'] - find Posts where post.blog === blog.$id
424
633
  // The backref tells us which field on the related type stores our ID
425
- // If no explicit backref, infer from schema relationships
426
- let backrefField = field.backref;
427
- if (!backrefField) {
428
- // Infer backref: look for a field on related entity that points to us
429
- for (const [relFieldName, relField] of relatedEntity.fields) {
430
- if (relField.isRelation &&
431
- relField.relatedType === typeName &&
432
- relField.direction !== 'backward' &&
433
- !relField.isArray) {
434
- // Found a forward single relation pointing to us - use its name
435
- backrefField = relFieldName;
436
- break;
634
+ const allResults = [];
635
+ for (const searchType of typesToSearch) {
636
+ const searchEntity = schema.entities.get(searchType);
637
+ if (!searchEntity)
638
+ continue;
639
+ // If no explicit backref, infer from schema relationships
640
+ let backrefField = field.backref;
641
+ if (!backrefField) {
642
+ // Infer backref: look for a field on related entity that points to us
643
+ for (const [relFieldName, relField] of searchEntity.fields) {
644
+ if (relField.isRelation &&
645
+ relField.relatedType === typeName &&
646
+ relField.direction !== 'backward' &&
647
+ !relField.isArray) {
648
+ // Found a forward single relation pointing to us - use its name
649
+ backrefField = relFieldName;
650
+ break;
651
+ }
652
+ }
653
+ // Fallback to entity name lowercase if no explicit relation found
654
+ if (!backrefField) {
655
+ backrefField = typeName.toLowerCase();
437
656
  }
438
657
  }
439
- // Fallback to entity name lowercase if no explicit relation found
440
- if (!backrefField) {
441
- backrefField = typeName.toLowerCase();
658
+ // Query the related type for entities that reference this entity
659
+ const results = await provider.list(searchType, {
660
+ where: { [backrefField]: id },
661
+ });
662
+ for (const r of results) {
663
+ const hydratedEntity = hydrateEntity(r, searchEntity, schema, resolveProvider);
664
+ allResults.push({ ...hydratedEntity, $matchedType: searchType });
442
665
  }
443
666
  }
444
- // Query the related type for entities that reference this entity
445
- const results = await provider.list(field.relatedType, {
446
- where: { [backrefField]: id },
447
- });
448
- return Promise.all(results.map((r) => hydrateEntity(r, relatedEntity, schema, resolveProvider)));
667
+ return allResults;
449
668
  }
450
669
  else if (field.isArray) {
451
670
  // Forward array relation - get related entities via relationship
@@ -458,9 +677,7 @@ export function hydrateEntity(data, entity, schema, resolveProvider) {
458
677
  if (!relatedId)
459
678
  return null;
460
679
  const result = await provider.get(field.relatedType, relatedId);
461
- return result
462
- ? hydrateEntity(result, relatedEntity, schema, resolveProvider)
463
- : null;
680
+ return result ? hydrateEntity(result, relatedEntity, schema, resolveProvider) : null;
464
681
  }
465
682
  })();
466
683
  },