@avytheone/efmesh 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +21 -0
- package/README.md +196 -0
- package/README.ru.md +203 -0
- package/SPEC.md +641 -0
- package/package.json +57 -0
- package/src/bin.ts +12 -0
- package/src/cli.ts +518 -0
- package/src/config.ts +44 -0
- package/src/core/audit.ts +88 -0
- package/src/core/errors.ts +27 -0
- package/src/core/graph.ts +76 -0
- package/src/core/interval.ts +119 -0
- package/src/core/model.ts +475 -0
- package/src/core/sql.ts +200 -0
- package/src/discovery.ts +77 -0
- package/src/efmesh.ts +94 -0
- package/src/engine/adapter.ts +52 -0
- package/src/engine/duckdb.ts +117 -0
- package/src/engine/postgres.ts +119 -0
- package/src/index.ts +114 -0
- package/src/init.ts +88 -0
- package/src/plan/audit-run.ts +101 -0
- package/src/plan/categorize.ts +41 -0
- package/src/plan/contract.ts +114 -0
- package/src/plan/diff.ts +36 -0
- package/src/plan/executor.ts +760 -0
- package/src/plan/fingerprint.ts +125 -0
- package/src/plan/graph-html.ts +157 -0
- package/src/plan/janitor.ts +130 -0
- package/src/plan/lineage.ts +143 -0
- package/src/plan/lock.ts +37 -0
- package/src/plan/metrics.ts +14 -0
- package/src/plan/naming.ts +91 -0
- package/src/plan/planner.ts +252 -0
- package/src/plan/run.ts +70 -0
- package/src/state/postgres.ts +396 -0
- package/src/state/sqlite.ts +407 -0
- package/src/state/store.ts +161 -0
- package/src/testing/index.ts +193 -0
package/src/discovery.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import * as NodePath from "node:path"
|
|
2
|
+
import { Data, Effect } from "effect"
|
|
3
|
+
import type { AnyModel } from "./core/model.ts"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Discovery моделей по glob (SPEC §12): файлы по маскам из конфига
|
|
7
|
+
* импортируются, все экспорты-модели (defineModel/defineExternal/defineSeed/
|
|
8
|
+
* defineSqlModel — у всех _tag: "Model") собираются в проект. Порядок —
|
|
9
|
+
* детерминированный (сортировка путей), два разных определения с одним
|
|
10
|
+
* именем — ошибка загрузки; один и тот же объект, реэкспортированный из
|
|
11
|
+
* нескольких файлов, считается один раз.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export class DiscoveryError extends Data.TaggedError("DiscoveryError")<{
|
|
15
|
+
readonly path: string
|
|
16
|
+
readonly reason: string
|
|
17
|
+
}> {}
|
|
18
|
+
|
|
19
|
+
export class DiscoveryConflictError extends Data.TaggedError("DiscoveryConflictError")<{
|
|
20
|
+
readonly name: string
|
|
21
|
+
readonly files: ReadonlyArray<string>
|
|
22
|
+
}> {}
|
|
23
|
+
|
|
24
|
+
const isModel = (value: unknown): value is AnyModel =>
|
|
25
|
+
typeof value === "object" &&
|
|
26
|
+
value !== null &&
|
|
27
|
+
(value as { _tag?: unknown })._tag === "Model" &&
|
|
28
|
+
typeof (value as { name?: { full?: unknown } }).name?.full === "string"
|
|
29
|
+
|
|
30
|
+
const scanPattern = (pattern: string, root: string): Effect.Effect<ReadonlyArray<string>, DiscoveryError> =>
|
|
31
|
+
Effect.tryPromise({
|
|
32
|
+
try: async () => {
|
|
33
|
+
const found: Array<string> = []
|
|
34
|
+
for await (const file of new Bun.Glob(pattern).scan({ cwd: root, absolute: true })) {
|
|
35
|
+
found.push(file)
|
|
36
|
+
}
|
|
37
|
+
return found
|
|
38
|
+
},
|
|
39
|
+
catch: (cause) => new DiscoveryError({ path: pattern, reason: String(cause) }),
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
export const discoverModels = (
|
|
43
|
+
patterns: string | ReadonlyArray<string>,
|
|
44
|
+
root: string,
|
|
45
|
+
): Effect.Effect<ReadonlyArray<AnyModel>, DiscoveryError | DiscoveryConflictError> =>
|
|
46
|
+
Effect.gen(function* () {
|
|
47
|
+
const masks = typeof patterns === "string" ? [patterns] : patterns
|
|
48
|
+
const files = new Set<string>()
|
|
49
|
+
for (const mask of masks) {
|
|
50
|
+
for (const file of yield* scanPattern(mask, root)) files.add(file)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const models: Array<AnyModel> = []
|
|
54
|
+
const seen = new Set<AnyModel>()
|
|
55
|
+
const owner = new Map<string, string>()
|
|
56
|
+
for (const file of [...files].sort()) {
|
|
57
|
+
const module = yield* Effect.tryPromise({
|
|
58
|
+
try: () => import(file) as Promise<Record<string, unknown>>,
|
|
59
|
+
catch: (cause) => new DiscoveryError({ path: file, reason: String(cause) }),
|
|
60
|
+
})
|
|
61
|
+
for (const value of Object.values(module)) {
|
|
62
|
+
if (!isModel(value) || seen.has(value)) continue
|
|
63
|
+
const relative = NodePath.relative(root, file)
|
|
64
|
+
const already = owner.get(value.name.full)
|
|
65
|
+
if (already !== undefined) {
|
|
66
|
+
return yield* new DiscoveryConflictError({
|
|
67
|
+
name: value.name.full,
|
|
68
|
+
files: [already, relative],
|
|
69
|
+
})
|
|
70
|
+
}
|
|
71
|
+
seen.add(value)
|
|
72
|
+
owner.set(value.name.full, relative)
|
|
73
|
+
models.push(value)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return models
|
|
77
|
+
})
|
package/src/efmesh.ts
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { Effect } from "effect"
|
|
2
|
+
import { buildGraph } from "./core/graph.ts"
|
|
3
|
+
import type { AnyModel } from "./core/model.ts"
|
|
4
|
+
import { render } from "./core/sql.ts"
|
|
5
|
+
import type { EngineError, SqlParseError } from "./engine/adapter.ts"
|
|
6
|
+
import { EngineAdapter } from "./engine/adapter.ts"
|
|
7
|
+
import { canonicalSql } from "./plan/fingerprint.ts"
|
|
8
|
+
import {
|
|
9
|
+
applyPlan,
|
|
10
|
+
type AppliedPlan,
|
|
11
|
+
type ApplyError,
|
|
12
|
+
type ApplyOptions,
|
|
13
|
+
} from "./plan/executor.ts"
|
|
14
|
+
import { envLockName, withStateLock, type LockHeldError, type LockOptions } from "./plan/lock.ts"
|
|
15
|
+
import {
|
|
16
|
+
planChanges,
|
|
17
|
+
type FingerprintVersionError,
|
|
18
|
+
type ForwardOnlyError,
|
|
19
|
+
type InvalidEnvironmentError,
|
|
20
|
+
type Plan,
|
|
21
|
+
type PlanOptions,
|
|
22
|
+
} from "./plan/planner.ts"
|
|
23
|
+
import type { SeedReadError } from "./core/errors.ts"
|
|
24
|
+
import type { GraphError } from "./core/graph.ts"
|
|
25
|
+
import type { StateError } from "./state/store.ts"
|
|
26
|
+
import { StateStore } from "./state/store.ts"
|
|
27
|
+
import { externalSourceRef, viewRef } from "./plan/naming.ts"
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Фасад efmesh (SPEC §10): обычные Effect'ы, встраиваемые в любое
|
|
31
|
+
* приложение; CLI — тонкая обёртка над ними.
|
|
32
|
+
*/
|
|
33
|
+
export const Efmesh = {
|
|
34
|
+
/** Посчитать план для окружения, ничего не меняя. Движок нужен для канонизации SQL. */
|
|
35
|
+
plan: (
|
|
36
|
+
env: string,
|
|
37
|
+
models: Iterable<AnyModel>,
|
|
38
|
+
options?: PlanOptions,
|
|
39
|
+
): Effect.Effect<
|
|
40
|
+
Plan,
|
|
41
|
+
| GraphError
|
|
42
|
+
| StateError
|
|
43
|
+
| InvalidEnvironmentError
|
|
44
|
+
| ForwardOnlyError
|
|
45
|
+
| FingerprintVersionError
|
|
46
|
+
| EngineError
|
|
47
|
+
| SqlParseError
|
|
48
|
+
| SeedReadError,
|
|
49
|
+
StateStore | EngineAdapter
|
|
50
|
+
> => buildGraph(models).pipe(Effect.flatMap((graph) => planChanges(env, graph, options))),
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* План + применение: физика, бэкфилл интервалов, view-слой, состояние.
|
|
54
|
+
* Идёт под межпроцессным локом `env:<имя>` (SPEC §14.6) — тем же, что у
|
|
55
|
+
* run: параллельные мутации окружения из разных процессов отсекаются.
|
|
56
|
+
*/
|
|
57
|
+
apply: (
|
|
58
|
+
env: string,
|
|
59
|
+
models: Iterable<AnyModel>,
|
|
60
|
+
options?: PlanOptions & ApplyOptions & LockOptions,
|
|
61
|
+
): Effect.Effect<AppliedPlan, ApplyError | LockHeldError, EngineAdapter | StateStore> =>
|
|
62
|
+
Effect.gen(function* () {
|
|
63
|
+
const graph = yield* buildGraph(models)
|
|
64
|
+
const plan = yield* planChanges(env, graph, options)
|
|
65
|
+
return yield* applyPlan(plan, graph, options)
|
|
66
|
+
}).pipe(withStateLock(envLockName(env), options?.lockTtlMs)),
|
|
67
|
+
|
|
68
|
+
/** Canonical-рендер SQL модели (ссылки — логические имена) для отладки. */
|
|
69
|
+
render: (models: Iterable<AnyModel>, name: string): Effect.Effect<string, GraphError> =>
|
|
70
|
+
buildGraph(models).pipe(Effect.map((graph) => canonicalSql(graph, name))),
|
|
71
|
+
|
|
72
|
+
/** Рендер SQL модели против view-слоя окружения — «как выполнит движок». */
|
|
73
|
+
renderFor: (
|
|
74
|
+
models: Iterable<AnyModel>,
|
|
75
|
+
name: string,
|
|
76
|
+
env: string,
|
|
77
|
+
): Effect.Effect<string, GraphError> =>
|
|
78
|
+
buildGraph(models).pipe(
|
|
79
|
+
Effect.map((graph) => {
|
|
80
|
+
const model = graph.models.get(name)
|
|
81
|
+
if (model === undefined) throw new Error(`модели ${name} нет в проекте`)
|
|
82
|
+
// view-слоя у external и embedded нет: источник как есть / подзапрос
|
|
83
|
+
const resolve = (ref: string): string => {
|
|
84
|
+
const source = graph.models.get(ref)!
|
|
85
|
+
if (source.kind._tag === "external") return externalSourceRef(source.kind.source)
|
|
86
|
+
if (source.kind._tag === "embedded") {
|
|
87
|
+
return `(${render(source.fragment, { resolveRef: resolve })})`
|
|
88
|
+
}
|
|
89
|
+
return viewRef(env, source.name)
|
|
90
|
+
}
|
|
91
|
+
return render(model.fragment, { resolveRef: resolve })
|
|
92
|
+
}),
|
|
93
|
+
),
|
|
94
|
+
} as const
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { Context, Data, Effect } from "effect"
|
|
2
|
+
|
|
3
|
+
export class EngineError extends Data.TaggedError("EngineError")<{
|
|
4
|
+
readonly sql: string
|
|
5
|
+
readonly cause: unknown
|
|
6
|
+
}> {}
|
|
7
|
+
|
|
8
|
+
/** Движок не смог распарсить SQL модели (SPEC §9.2). */
|
|
9
|
+
export class SqlParseError extends Data.TaggedError("SqlParseError")<{
|
|
10
|
+
readonly sql: string
|
|
11
|
+
readonly message: string
|
|
12
|
+
}> {}
|
|
13
|
+
|
|
14
|
+
export interface EngineColumn {
|
|
15
|
+
readonly name: string
|
|
16
|
+
readonly type: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Адаптер движка (SPEC §9.1). Минимальная поверхность: DDL-помощники
|
|
21
|
+
* живут отдельно (executor) и ходят через execute — адаптеру незачем
|
|
22
|
+
* знать про физический/виртуальный слой.
|
|
23
|
+
*/
|
|
24
|
+
export type Dialect = "duckdb" | "postgres"
|
|
25
|
+
|
|
26
|
+
export interface Engine {
|
|
27
|
+
readonly dialect: Dialect
|
|
28
|
+
readonly query: (
|
|
29
|
+
sql: string,
|
|
30
|
+
) => Effect.Effect<ReadonlyArray<Record<string, unknown>>, EngineError>
|
|
31
|
+
readonly execute: (sql: string) => Effect.Effect<void, EngineError>
|
|
32
|
+
/**
|
|
33
|
+
* Набор стейтментов одной транзакцией движка, откат при любой ошибке.
|
|
34
|
+
* Примитив адаптера, а не BEGIN/COMMIT через execute: на пуле соединений
|
|
35
|
+
* (Postgres) отдельные вызовы разъехались бы по разным соединениям.
|
|
36
|
+
*/
|
|
37
|
+
readonly transaction: (
|
|
38
|
+
statements: ReadonlyArray<string>,
|
|
39
|
+
) => Effect.Effect<void, EngineError>
|
|
40
|
+
/** Имена и типы колонок запроса без его выполнения (контракт схемы, SPEC §3.2). */
|
|
41
|
+
readonly describe: (sql: string) => Effect.Effect<ReadonlyArray<EngineColumn>, EngineError>
|
|
42
|
+
/**
|
|
43
|
+
* Канонический вид SELECT-запроса для fingerprint (SPEC §4, §9.2):
|
|
44
|
+
* парсинг родным парсером движка, нормализация, детерминированная
|
|
45
|
+
* сериализация. Переформатирование текста не меняет результат.
|
|
46
|
+
*/
|
|
47
|
+
readonly canonicalize: (sql: string) => Effect.Effect<string, EngineError | SqlParseError>
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export class EngineAdapter extends Context.Service<EngineAdapter, Engine>()(
|
|
51
|
+
"efmesh/EngineAdapter",
|
|
52
|
+
) {}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { DuckDBInstance } from "@duckdb/node-api"
|
|
2
|
+
import { Effect, Layer, Semaphore } from "effect"
|
|
3
|
+
import type { Engine, EngineColumn } from "./adapter.ts"
|
|
4
|
+
import { EngineAdapter, EngineError, SqlParseError } from "./adapter.ts"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* AST от json_serialize_sql содержит позиции токенов (query_location) —
|
|
8
|
+
* единственное, чем отличаются одинаковые по смыслу, но по-разному
|
|
9
|
+
* отформатированные запросы. Вычищаем — остаётся формат-инвариантное ядро.
|
|
10
|
+
*/
|
|
11
|
+
const stripLocations = (value: unknown): unknown => {
|
|
12
|
+
if (Array.isArray(value)) return value.map(stripLocations)
|
|
13
|
+
if (value !== null && typeof value === "object") {
|
|
14
|
+
const out: Record<string, unknown> = {}
|
|
15
|
+
for (const [key, inner] of Object.entries(value)) {
|
|
16
|
+
if (key === "query_location") continue
|
|
17
|
+
out[key] = stripLocations(inner)
|
|
18
|
+
}
|
|
19
|
+
return out
|
|
20
|
+
}
|
|
21
|
+
return value
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface DuckDBEngineOptions {
|
|
25
|
+
/** Путь к файлу базы; по умолчанию in-memory. */
|
|
26
|
+
readonly path?: string
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export const DuckDBEngineLive = (
|
|
30
|
+
options?: DuckDBEngineOptions,
|
|
31
|
+
): Layer.Layer<EngineAdapter, EngineError> =>
|
|
32
|
+
Layer.effect(
|
|
33
|
+
EngineAdapter,
|
|
34
|
+
Effect.gen(function* () {
|
|
35
|
+
const path = options?.path ?? ":memory:"
|
|
36
|
+
const { connection } = yield* Effect.acquireRelease(
|
|
37
|
+
Effect.tryPromise({
|
|
38
|
+
try: async () => {
|
|
39
|
+
const instance = await DuckDBInstance.create(path)
|
|
40
|
+
const connection = await instance.connect()
|
|
41
|
+
return { instance, connection }
|
|
42
|
+
},
|
|
43
|
+
catch: (cause) => new EngineError({ sql: `<connect ${path}>`, cause }),
|
|
44
|
+
}),
|
|
45
|
+
({ connection, instance }) =>
|
|
46
|
+
Effect.sync(() => {
|
|
47
|
+
connection.closeSync()
|
|
48
|
+
instance.closeSync()
|
|
49
|
+
}),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
const run = (sqlText: string) =>
|
|
53
|
+
Effect.tryPromise({
|
|
54
|
+
try: () => connection.run(sqlText),
|
|
55
|
+
catch: (cause) => new EngineError({ sql: sqlText, cause }),
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
const query: Engine["query"] = (sqlText) =>
|
|
59
|
+
run(sqlText).pipe(
|
|
60
|
+
Effect.flatMap((result) =>
|
|
61
|
+
Effect.tryPromise({
|
|
62
|
+
try: () => result.getRowObjects(),
|
|
63
|
+
catch: (cause) => new EngineError({ sql: sqlText, cause }),
|
|
64
|
+
}),
|
|
65
|
+
),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
// одно соединение: параллельные транзакции сериализуются семафором,
|
|
69
|
+
// чтобы фибры не перемежали чужие BEGIN/COMMIT
|
|
70
|
+
const transactionLock = yield* Semaphore.make(1)
|
|
71
|
+
|
|
72
|
+
const service: Engine = {
|
|
73
|
+
dialect: "duckdb",
|
|
74
|
+
query,
|
|
75
|
+
execute: (sqlText) => Effect.asVoid(run(sqlText)),
|
|
76
|
+
transaction: (statements) =>
|
|
77
|
+
transactionLock.withPermits(1)(
|
|
78
|
+
Effect.asVoid(run("BEGIN")).pipe(
|
|
79
|
+
Effect.andThen(
|
|
80
|
+
Effect.forEach(statements, (statement) => run(statement), { discard: true }),
|
|
81
|
+
),
|
|
82
|
+
Effect.andThen(Effect.asVoid(run("COMMIT"))),
|
|
83
|
+
Effect.onError(() => Effect.asVoid(run("ROLLBACK")).pipe(Effect.ignore)),
|
|
84
|
+
),
|
|
85
|
+
),
|
|
86
|
+
describe: (sqlText) =>
|
|
87
|
+
query(`DESCRIBE (${sqlText})`).pipe(
|
|
88
|
+
Effect.map((rows): ReadonlyArray<EngineColumn> =>
|
|
89
|
+
rows.map((row) => ({
|
|
90
|
+
name: String(row["column_name"]),
|
|
91
|
+
type: String(row["column_type"]),
|
|
92
|
+
})),
|
|
93
|
+
),
|
|
94
|
+
),
|
|
95
|
+
canonicalize: (sqlText) =>
|
|
96
|
+
query(
|
|
97
|
+
`SELECT json_serialize_sql('${sqlText.replaceAll(`'`, `''`)}') AS ast`,
|
|
98
|
+
).pipe(
|
|
99
|
+
Effect.flatMap((rows) => {
|
|
100
|
+
const ast = JSON.parse(String(rows[0]?.["ast"])) as {
|
|
101
|
+
readonly error: boolean
|
|
102
|
+
readonly error_message?: string
|
|
103
|
+
}
|
|
104
|
+
// ошибка парсинга приходит содержимым JSON, не исключением
|
|
105
|
+
if (ast.error) {
|
|
106
|
+
return new SqlParseError({
|
|
107
|
+
sql: sqlText,
|
|
108
|
+
message: ast.error_message ?? "parse error",
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
return Effect.succeed(JSON.stringify(stripLocations(ast)))
|
|
112
|
+
}),
|
|
113
|
+
),
|
|
114
|
+
}
|
|
115
|
+
return service
|
|
116
|
+
}),
|
|
117
|
+
)
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { SQL } from "bun"
|
|
2
|
+
import { Effect, Layer } from "effect"
|
|
3
|
+
import { parse } from "libpg-query"
|
|
4
|
+
import type { Engine, EngineColumn } from "./adapter.ts"
|
|
5
|
+
import { EngineAdapter, EngineError, SqlParseError } from "./adapter.ts"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Postgres-адаптер (SPEC §9.1, F3) поверх встроенного клиента Bun (`Bun.SQL`):
|
|
9
|
+
* пул соединений из коробки — транзакция закрепляется за одним соединением
|
|
10
|
+
* через sql.begin, остальные запросы свободно распределяются по пулу,
|
|
11
|
+
* что и даёт честный параллелизм бэкфилла (SPEC §5.3).
|
|
12
|
+
*
|
|
13
|
+
* Канонизация — libpg_query (WASM-сборка парсера самого Postgres, SPEC §9.2):
|
|
14
|
+
* дерево разбора с вычищенными позициями токенов формат-инвариантно,
|
|
15
|
+
* как и json_serialize_sql у DuckDB.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Позиции токенов (location) — единственная формат-зависимость дерева libpg_query. */
|
|
19
|
+
const stripLocations = (value: unknown): unknown => {
|
|
20
|
+
if (Array.isArray(value)) return value.map(stripLocations)
|
|
21
|
+
if (value !== null && typeof value === "object") {
|
|
22
|
+
const out: Record<string, unknown> = {}
|
|
23
|
+
for (const [key, inner] of Object.entries(value)) {
|
|
24
|
+
if (key === "location") continue
|
|
25
|
+
out[key] = stripLocations(inner)
|
|
26
|
+
}
|
|
27
|
+
return out
|
|
28
|
+
}
|
|
29
|
+
return value
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Канонизация Postgres-SQL — чистая (libpg_query, сервер не нужен);
|
|
34
|
+
* вынесена из Layer, чтобы golden-тесты могли заморозить канон
|
|
35
|
+
* независимо от пула (SPEC §4: канонизация = контракт fingerprint).
|
|
36
|
+
*/
|
|
37
|
+
export const canonicalizePostgresSql = (
|
|
38
|
+
sqlText: string,
|
|
39
|
+
): Effect.Effect<string, SqlParseError> => {
|
|
40
|
+
// $start/$end канонического рендера — не синтаксис Postgres;
|
|
41
|
+
// детерминированная замена на $1/$2 сохраняет стабильность fingerprint
|
|
42
|
+
const parameterized = sqlText.replace(/\$start\b/g, "$1").replace(/\$end\b/g, "$2")
|
|
43
|
+
return Effect.tryPromise({
|
|
44
|
+
try: () => parse(parameterized),
|
|
45
|
+
catch: (cause) =>
|
|
46
|
+
new SqlParseError({
|
|
47
|
+
sql: sqlText,
|
|
48
|
+
message: cause instanceof Error ? cause.message : String(cause),
|
|
49
|
+
}),
|
|
50
|
+
}).pipe(Effect.map((tree) => JSON.stringify(stripLocations(tree))))
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface PostgresEngineOptions {
|
|
54
|
+
/** postgres://… или unix-сокет через ?host=/путь. */
|
|
55
|
+
readonly url: string
|
|
56
|
+
/** Размер пула соединений; по умолчанию 8. */
|
|
57
|
+
readonly max?: number
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export const PostgresEngineLive = (
|
|
61
|
+
options: PostgresEngineOptions,
|
|
62
|
+
): Layer.Layer<EngineAdapter, EngineError> =>
|
|
63
|
+
Layer.effect(
|
|
64
|
+
EngineAdapter,
|
|
65
|
+
Effect.gen(function* () {
|
|
66
|
+
const sql = yield* Effect.acquireRelease(
|
|
67
|
+
Effect.try({
|
|
68
|
+
try: () => new SQL({ url: options.url, max: options.max ?? 8 }),
|
|
69
|
+
catch: (cause) => new EngineError({ sql: `<connect ${options.url}>`, cause }),
|
|
70
|
+
}),
|
|
71
|
+
(pool) => Effect.promise(() => pool.end()).pipe(Effect.ignore),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
const query: Engine["query"] = (sqlText) =>
|
|
75
|
+
Effect.tryPromise({
|
|
76
|
+
try: async () => (await sql.unsafe(sqlText)) as ReadonlyArray<Record<string, unknown>>,
|
|
77
|
+
catch: (cause) => new EngineError({ sql: sqlText, cause }),
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
const service: Engine = {
|
|
81
|
+
dialect: "postgres",
|
|
82
|
+
query,
|
|
83
|
+
execute: (sqlText) => Effect.asVoid(query(sqlText)),
|
|
84
|
+
transaction: (statements) =>
|
|
85
|
+
Effect.tryPromise({
|
|
86
|
+
try: () =>
|
|
87
|
+
sql.begin(async (tx) => {
|
|
88
|
+
for (const statement of statements) await tx.unsafe(statement)
|
|
89
|
+
}),
|
|
90
|
+
catch: (cause) => new EngineError({ sql: statements.join(";\n"), cause }),
|
|
91
|
+
}).pipe(Effect.asVoid),
|
|
92
|
+
// DESCRIBE у Postgres нет: временный view на одном соединении
|
|
93
|
+
// (sql.begin) даёт имена и типы из каталога без выполнения запроса
|
|
94
|
+
describe: (sqlText) =>
|
|
95
|
+
Effect.tryPromise({
|
|
96
|
+
try: () =>
|
|
97
|
+
sql.begin(async (tx) => {
|
|
98
|
+
await tx.unsafe(`CREATE TEMP VIEW __efmesh_describe AS ${sqlText}`)
|
|
99
|
+
const rows = (await tx.unsafe(`
|
|
100
|
+
SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS type
|
|
101
|
+
FROM pg_attribute a
|
|
102
|
+
WHERE a.attrelid = '__efmesh_describe'::regclass
|
|
103
|
+
AND a.attnum > 0 AND NOT a.attisdropped
|
|
104
|
+
ORDER BY a.attnum
|
|
105
|
+
`)) as ReadonlyArray<{ name: string; type: string }>
|
|
106
|
+
await tx.unsafe(`DROP VIEW __efmesh_describe`)
|
|
107
|
+
return rows
|
|
108
|
+
}),
|
|
109
|
+
catch: (cause) => new EngineError({ sql: sqlText, cause }),
|
|
110
|
+
}).pipe(
|
|
111
|
+
Effect.map((rows): ReadonlyArray<EngineColumn> =>
|
|
112
|
+
rows.map((row) => ({ name: row.name, type: row.type })),
|
|
113
|
+
),
|
|
114
|
+
),
|
|
115
|
+
canonicalize: canonicalizePostgresSql,
|
|
116
|
+
}
|
|
117
|
+
return service
|
|
118
|
+
}),
|
|
119
|
+
)
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Публичное API efmesh (F6): осознанный whitelist вместо `export *` —
|
|
3
|
+
* всё, что экспортировано здесь, становится semver-обязательством пакета.
|
|
4
|
+
* Внутренности (naming, lock-хелперы, executor, planner, fingerprint-кухня)
|
|
5
|
+
* намеренно не экспортируются; юнит-тесты моделей — `efmesh/testing`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// — определение проекта: модели, виды, аудиты, конфиг —
|
|
9
|
+
export {
|
|
10
|
+
defineExternal,
|
|
11
|
+
defineModel,
|
|
12
|
+
defineSeed,
|
|
13
|
+
defineSqlModel,
|
|
14
|
+
external,
|
|
15
|
+
kind,
|
|
16
|
+
columnNames,
|
|
17
|
+
parseModelName,
|
|
18
|
+
type AnyModel,
|
|
19
|
+
type ExternalConfig,
|
|
20
|
+
type ExternalSource,
|
|
21
|
+
type IncrementalByTimeRangeOptions,
|
|
22
|
+
type MaterializationTarget,
|
|
23
|
+
type Model,
|
|
24
|
+
type ModelConfig,
|
|
25
|
+
type ModelCtx,
|
|
26
|
+
type ModelKind,
|
|
27
|
+
type ModelName,
|
|
28
|
+
} from "./core/model.ts"
|
|
29
|
+
export { audit, type Audit, type AuditCtx } from "./core/audit.ts"
|
|
30
|
+
export { defineConfig, type EfmeshConfig } from "./config.ts"
|
|
31
|
+
export { discoverModels, DiscoveryConflictError, DiscoveryError } from "./discovery.ts"
|
|
32
|
+
|
|
33
|
+
// — фасад и операции: обычные Effect'ы для встраивания (SPEC §10) —
|
|
34
|
+
export { Efmesh } from "./efmesh.ts"
|
|
35
|
+
export { run, daemon, Runner, RunBlockedByChangesError, type RunError, type RunOptions } from "./plan/run.ts"
|
|
36
|
+
export { janitor, type JanitorOptions, type JanitorReport } from "./plan/janitor.ts"
|
|
37
|
+
export {
|
|
38
|
+
auditEnvironment,
|
|
39
|
+
AuditTargetError,
|
|
40
|
+
EnvironmentAuditError,
|
|
41
|
+
type AuditRunReport,
|
|
42
|
+
type AuditRunResult,
|
|
43
|
+
} from "./plan/audit-run.ts"
|
|
44
|
+
export { diffEnvironments, type EnvDiff } from "./plan/diff.ts"
|
|
45
|
+
export { formatLineage, lineage, LineageError, type LineageNode } from "./plan/lineage.ts"
|
|
46
|
+
|
|
47
|
+
// — план и применение: типы результата и опций —
|
|
48
|
+
export type {
|
|
49
|
+
ChangeCategory,
|
|
50
|
+
Plan,
|
|
51
|
+
PlanAction,
|
|
52
|
+
PlanOptions,
|
|
53
|
+
} from "./plan/planner.ts"
|
|
54
|
+
export { FingerprintVersionError, ForwardOnlyError, InvalidEnvironmentError } from "./plan/planner.ts"
|
|
55
|
+
export type { AppliedPlan, ApplyError, ApplyOptions } from "./plan/executor.ts"
|
|
56
|
+
export {
|
|
57
|
+
AttachNotConfiguredError,
|
|
58
|
+
DucklakeNotConfiguredError,
|
|
59
|
+
EngineFeatureError,
|
|
60
|
+
LakeNotConfiguredError,
|
|
61
|
+
} from "./plan/executor.ts"
|
|
62
|
+
export { AuditFailure } from "./core/audit.ts"
|
|
63
|
+
export { SchemaMismatchError } from "./plan/contract.ts"
|
|
64
|
+
export { LockHeldError, type LockOptions } from "./plan/lock.ts"
|
|
65
|
+
export { FINGERPRINT_VERSION } from "./plan/fingerprint.ts"
|
|
66
|
+
|
|
67
|
+
// — граф: составные ошибки загрузки проекта —
|
|
68
|
+
export {
|
|
69
|
+
DagCycleError,
|
|
70
|
+
DuplicateModelError,
|
|
71
|
+
ModelDefinitionError,
|
|
72
|
+
SeedReadError,
|
|
73
|
+
UnknownDependencyError,
|
|
74
|
+
} from "./core/errors.ts"
|
|
75
|
+
export type { GraphError } from "./core/graph.ts"
|
|
76
|
+
|
|
77
|
+
// — движки: слои и сервис (кастомные обёртки — например, для тестов) —
|
|
78
|
+
export {
|
|
79
|
+
EngineAdapter,
|
|
80
|
+
EngineError,
|
|
81
|
+
SqlParseError,
|
|
82
|
+
type Dialect,
|
|
83
|
+
type Engine,
|
|
84
|
+
type EngineColumn,
|
|
85
|
+
} from "./engine/adapter.ts"
|
|
86
|
+
export { DuckDBEngineLive, type DuckDBEngineOptions } from "./engine/duckdb.ts"
|
|
87
|
+
export {
|
|
88
|
+
canonicalizePostgresSql,
|
|
89
|
+
PostgresEngineLive,
|
|
90
|
+
type PostgresEngineOptions,
|
|
91
|
+
} from "./engine/postgres.ts"
|
|
92
|
+
|
|
93
|
+
// — state store: слои, миграции, записи —
|
|
94
|
+
export {
|
|
95
|
+
StateStore,
|
|
96
|
+
StateError,
|
|
97
|
+
StateSchemaError,
|
|
98
|
+
STATE_VERSION,
|
|
99
|
+
type EnvironmentRecord,
|
|
100
|
+
type IntervalRecord,
|
|
101
|
+
type MigrationReport,
|
|
102
|
+
type PlanRecord,
|
|
103
|
+
type SnapshotRecord,
|
|
104
|
+
type StateStoreShape,
|
|
105
|
+
} from "./state/store.ts"
|
|
106
|
+
export { migrateSqliteState, SqliteStateLive, type SqliteStateOptions } from "./state/sqlite.ts"
|
|
107
|
+
export {
|
|
108
|
+
migratePostgresState,
|
|
109
|
+
PostgresStateLive,
|
|
110
|
+
type PostgresStateOptions,
|
|
111
|
+
} from "./state/postgres.ts"
|
|
112
|
+
|
|
113
|
+
// — интервалы: границы для options.now и разбор отчётов —
|
|
114
|
+
export { fromIso, toIso, type Interval, type IntervalUnit } from "./core/interval.ts"
|
package/src/init.ts
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs"
|
|
2
|
+
import * as NodePath from "node:path"
|
|
3
|
+
import { Data, Effect } from "effect"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* `efmesh init` (SPEC §12): скаффолд минимального проекта — конфиг,
|
|
7
|
+
* пара моделей (seed + витрина с аудитом) и данные к ним. Ничего не
|
|
8
|
+
* перезаписывает: существующий efmesh.config.ts — честная ошибка.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export class InitError extends Data.TaggedError("InitError")<{
|
|
12
|
+
readonly path: string
|
|
13
|
+
readonly reason: string
|
|
14
|
+
}> {}
|
|
15
|
+
|
|
16
|
+
const MODELS_TS = `import { Schema } from "effect"
|
|
17
|
+
import { audit, defineModel, defineSeed, kind } from "@avytheone/efmesh"
|
|
18
|
+
|
|
19
|
+
// seed: справочник из файла; содержимое входит в fingerprint —
|
|
20
|
+
// правка CSV = новая версия и пересборка
|
|
21
|
+
export const departments = defineSeed({
|
|
22
|
+
name: "ref.departments",
|
|
23
|
+
file: "seeds/departments.csv",
|
|
24
|
+
schema: Schema.Struct({ dept: Schema.String, floor: Schema.Number }),
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
// витрина: full-модель с контрактом схемы и blocking-аудитом
|
|
28
|
+
export const floors = defineModel(
|
|
29
|
+
{
|
|
30
|
+
name: "mart.floors",
|
|
31
|
+
kind: kind.full(),
|
|
32
|
+
schema: Schema.Struct({ floor: Schema.Number, depts: Schema.Number }),
|
|
33
|
+
audits: [audit.notNull("floor")],
|
|
34
|
+
},
|
|
35
|
+
(ctx) => ctx.sql\`
|
|
36
|
+
SELECT floor, count(*)::INT AS depts
|
|
37
|
+
FROM \${ctx.ref(departments)}
|
|
38
|
+
GROUP BY floor
|
|
39
|
+
\`,
|
|
40
|
+
)
|
|
41
|
+
`
|
|
42
|
+
|
|
43
|
+
const CONFIG_TS = `import { defineConfig } from "@avytheone/efmesh"
|
|
44
|
+
import { departments, floors } from "./models.ts"
|
|
45
|
+
|
|
46
|
+
export default defineConfig({
|
|
47
|
+
models: [departments, floors],
|
|
48
|
+
// engine: { url: "postgres://…" }, // вместо DuckDB-файла
|
|
49
|
+
// lake: { path: "lake" }, // для target: "parquet"
|
|
50
|
+
})
|
|
51
|
+
`
|
|
52
|
+
|
|
53
|
+
const SEED_CSV = `dept,floor
|
|
54
|
+
ОРИТ,3
|
|
55
|
+
терапия,2
|
|
56
|
+
хирургия,3
|
|
57
|
+
`
|
|
58
|
+
|
|
59
|
+
export const scaffold = (dir: string): Effect.Effect<ReadonlyArray<string>, InitError> =>
|
|
60
|
+
Effect.gen(function* () {
|
|
61
|
+
const root = NodePath.resolve(dir)
|
|
62
|
+
const files: ReadonlyArray<readonly [string, string]> = [
|
|
63
|
+
["efmesh.config.ts", CONFIG_TS],
|
|
64
|
+
["models.ts", MODELS_TS],
|
|
65
|
+
["seeds/departments.csv", SEED_CSV],
|
|
66
|
+
]
|
|
67
|
+
for (const [relative] of files) {
|
|
68
|
+
if (existsSync(NodePath.join(root, relative))) {
|
|
69
|
+
return yield* new InitError({
|
|
70
|
+
path: NodePath.join(root, relative),
|
|
71
|
+
reason: "файл уже существует — init ничего не перезаписывает",
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return yield* Effect.try({
|
|
76
|
+
try: () => {
|
|
77
|
+
const created: Array<string> = []
|
|
78
|
+
for (const [relative, content] of files) {
|
|
79
|
+
const path = NodePath.join(root, relative)
|
|
80
|
+
mkdirSync(NodePath.dirname(path), { recursive: true })
|
|
81
|
+
writeFileSync(path, content)
|
|
82
|
+
created.push(relative)
|
|
83
|
+
}
|
|
84
|
+
return created
|
|
85
|
+
},
|
|
86
|
+
catch: (cause) => new InitError({ path: root, reason: String(cause) }),
|
|
87
|
+
})
|
|
88
|
+
})
|