create-windy 0.5.1 → 0.6.0

package/template/apps/document-server/src/southwind_document_server/multipart_output.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import tempfile
+from email.message import Message
+from pathlib import Path
+from typing import IO, cast
+
+import httpx
+
+from .adapter_errors import invalid_output
+from .artifacts import AdapterExecution, ProducedArtifact
+from .contracts import JsonValue, OutputArtifactSinkDescriptor
+from .output_validation import validate_output
+
+MANIFEST_MAX_BYTES = 64 * 1024
+FRAMING_MAX_BYTES = 128 * 1024
+HEADER_MAX_BYTES = 8 * 1024
+BOUNDARY_PATTERN = re.compile(r"^[A-Za-z0-9'()+_,./:=?-]{1,70}$")
+HEADER_NAME_PATTERN = re.compile(r"^[a-z0-9-]{1,64}$")
+
+
+async def parse_adapter_response(
+    response: httpx.Response,
+    max_output_bytes: int,
+    max_units: int,
+    sink: OutputArtifactSinkDescriptor | None,
+) -> AdapterExecution:
+    content_type = response.headers.get("content-type", "")
+    media_type, boundary = parse_content_type(content_type)
+    if media_type == "application/json":
+        reader = _BodyReader(response, max_output_bytes)
+        value = decode_json(await reader.read_all())
+        validate_output(value, max_units)
+        return AdapterExecution(value)
+    if media_type != "multipart/related" or boundary is None or sink is None:
+        raise invalid_output("Document Adapter Content-Type 无效")
+    reader = _BodyReader(
+        response,
+        max_output_bytes + FRAMING_MAX_BYTES,
+    )
+    return await parse_multipart(reader, boundary, max_output_bytes, max_units, sink)
+
+
+async def parse_multipart(
+    reader: _BodyReader,
+    boundary: str,
+    max_output_bytes: int,
+    max_units: int,
+    sink: OutputArtifactSinkDescriptor,
+) -> AdapterExecution:
+    marker = f"--{boundary}".encode()
+    if await reader.read_line() != marker + b"\r\n":
+        raise invalid_output("Document multipart 起始边界无效")
+    headers = await read_headers(reader)
+    manifest_length = content_length(headers, min(max_output_bytes, MANIFEST_MAX_BYTES))
+    if content_media_type(headers) != "application/json":
+        raise invalid_output("Document multipart manifest 类型无效")
+    value = decode_json(await reader.read_exact(manifest_length))
+    declarations = validate_output(value, max_units, sink)
+    if sum(item.byte_size for item in declarations) > (
+        max_output_bytes - manifest_length
+    ):
+        raise invalid_output("Document Adapter 输出超过总上限")
+    closed = await read_boundary(reader, marker)
+    if closed != (not declarations):
+        raise invalid_output("Document multipart part 数量无效")
+    produced: list[ProducedArtifact] = []
+    try:
+        for index, declaration in enumerate(declarations):
+            headers = await read_headers(reader)
+            if content_media_type(headers) != declaration.media_type:
+                raise invalid_output("Document Artifact MIME 不一致")
+            if headers.get("content-id") != f"<{declaration.output_key}>":
+                raise invalid_output("Document Artifact Content-ID 不一致")
+            length = content_length(headers, sink.limits.max_artifact_bytes)
+            if length != declaration.byte_size:
+                raise invalid_output("Document Artifact 长度不一致")
+            path: Path | None = None
+            try:
+                with tempfile.NamedTemporaryFile(delete=False) as target:
+                    path = Path(target.name)
+                    digest = await reader.copy_exact(length, target)
+                if digest != declaration.sha256:
+                    raise invalid_output("Document Artifact hash 不一致")
+                produced.append(ProducedArtifact(declaration, path))
+                closed = await read_boundary(reader, marker)
+                if closed != (index == len(declarations) - 1):
+                    raise invalid_output("Document multipart 结束边界无效")
+            except BaseException:
+                if path is not None:
+                    path.unlink(missing_ok=True)
+                raise
+        await reader.require_eof()
+        return AdapterExecution(value, tuple(produced))
+    except BaseException:
+        for artifact in produced:
+            artifact.path.unlink(missing_ok=True)
+        raise
+
+
+class _BodyReader:
+    def __init__(self, response: httpx.Response, max_bytes: int) -> None:
+        self._iterator = response.aiter_bytes(chunk_size=64 * 1024).__aiter__()
+        self._buffer = bytearray()
+        self._max_bytes = max_bytes
+        self._received = 0
+        self._eof = False
+
+    async def read_exact(self, size: int) -> bytes:
+        if size < 0:
+            raise invalid_output("Document Adapter 长度无效")
+        await self._fill(size)
+        if len(self._buffer) < size:
+            raise invalid_output("Document Adapter 正文提前结束")
+        value = bytes(self._buffer[:size])
+        del self._buffer[:size]
+        return value
+
+    async def read_line(self) -> bytes:
+        while True:
+            index = self._buffer.find(b"\r\n")
+            if index >= 0:
+                if index + 2 > HEADER_MAX_BYTES:
+                    raise invalid_output("Document multipart 行过长")
+                value = bytes(self._buffer[: index + 2])
+                del self._buffer[: index + 2]
+                return value
+            if len(self._buffer) >= HEADER_MAX_BYTES or self._eof:
+                raise invalid_output("Document multipart 行无效")
+            await self._fill(len(self._buffer) + 1)
+
+    async def copy_exact(self, size: int, target: IO[bytes]) -> str:
+        remaining = size
+        digest = hashlib.sha256()
+        while remaining:
+            await self._fill(1)
+            if not self._buffer:
+                raise invalid_output("Document Artifact 正文提前结束")
+            count = min(remaining, len(self._buffer), 64 * 1024)
+            chunk = bytes(self._buffer[:count])
+            del self._buffer[:count]
+            target.write(chunk)
+            digest.update(chunk)
+            remaining -= count
+        return digest.hexdigest()
+
+    async def read_all(self) -> bytes:
+        while not self._eof:
+            await self._fill(len(self._buffer) + 1)
+        return bytes(self._buffer)
+
+    async def require_eof(self) -> None:
+        if self._buffer:
+            raise invalid_output("Document multipart 包含尾随数据")
+        await self._fill(1)
+        if self._buffer:
+            raise invalid_output("Document multipart 包含尾随数据")
+
+    async def _fill(self, minimum: int) -> None:
+        while len(self._buffer) < minimum and not self._eof:
+            try:
+                chunk = await anext(self._iterator)
+            except StopAsyncIteration:
+                self._eof = True
+                break
+            self._received += len(chunk)
+            if self._received > self._max_bytes:
+                raise invalid_output("Document Adapter 输出超过上限")
+            self._buffer.extend(chunk)
+
+
+async def read_headers(reader: _BodyReader) -> dict[str, str]:
+    headers: dict[str, str] = {}
+    total = 0
+    while True:
+        line = await reader.read_line()
+        total += len(line)
+        if total > HEADER_MAX_BYTES:
+            raise invalid_output("Document multipart Header 超过上限")
+        if line == b"\r\n":
+            return headers
+        try:
+            name_raw, value_raw = line[:-2].decode("ascii").split(":", 1)
+        except (UnicodeDecodeError, ValueError):
+            raise invalid_output("Document multipart Header 无效") from None
+        name = name_raw.lower()
+        value = value_raw.strip()
+        if not HEADER_NAME_PATTERN.fullmatch(name) or name in headers or not value:
+            raise invalid_output("Document multipart Header 无效")
+        headers[name] = value
+
+
+async def read_boundary(reader: _BodyReader, marker: bytes) -> bool:
+    if await reader.read_exact(2) != b"\r\n":
+        raise invalid_output("Document multipart part 边界无效")
+    line = await reader.read_line()
+    if line == marker + b"\r\n":
+        return False
+    if line == marker + b"--\r\n":
+        return True
+    raise invalid_output("Document multipart part 边界无效")
+
+
+def content_length(headers: dict[str, str], maximum: int) -> int:
+    raw = headers.get("content-length", "")
+    if not re.fullmatch(r"0|[1-9][0-9]*", raw):
+        raise invalid_output("Document multipart Content-Length 无效")
+    value = int(raw)
+    if value > maximum:
+        raise invalid_output("Document multipart part 超过上限")
+    if "content-transfer-encoding" in headers:
+        raise invalid_output("Document multipart 禁止内容编码")
+    return value
+
+
+def content_media_type(headers: dict[str, str]) -> str:
+    media_type, boundary = parse_content_type(headers.get("content-type", ""))
+    if boundary is not None:
+        raise invalid_output("Document multipart part 类型无效")
+    return media_type
+
+
+def parse_content_type(raw: str) -> tuple[str, str | None]:
+    message = Message()
+    message["content-type"] = raw
+    media_type = message.get_content_type().lower()
+    boundary = message.get_param("boundary")
+    if boundary is not None and (
+        not isinstance(boundary, str) or not BOUNDARY_PATTERN.fullmatch(boundary)
+    ):
+        raise invalid_output("Document multipart boundary 无效")
+    return media_type, boundary
+
+
+def decode_json(raw: bytes) -> JsonValue:
+    if not raw:
+        raise invalid_output("Document Adapter JSON 为空")
+    try:
+        text = raw.decode("utf-8", errors="strict")
+        return cast(
+            JsonValue,
+            json.loads(
+                text,
+                parse_constant=reject_constant,
+                object_pairs_hook=unique_object,
+            ),
+        )
+    except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
+        raise invalid_output("Document Adapter JSON 无效") from None
+
+
+def reject_constant(_value: str) -> None:
+    raise ValueError
+
+
+def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    value: dict[str, object] = {}
+    for key, item in pairs:
+        if key in value:
+            raise ValueError
+        value[key] = item
+    return value

package/template/apps/document-server/src/southwind_document_server/output_validation.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import math
+from typing import TypeGuard
+
+from pydantic import ValidationError
+
+from .adapter_errors import SafeDocumentError, invalid_output
+from .contracts import (
+    JsonValue,
+    OutputArtifactDeclaration,
+    OutputArtifactReference,
+    OutputArtifactSinkDescriptor,
+)
+
+
+def validate_output(
+    value: JsonValue,
+    max_units: int,
+    sink: OutputArtifactSinkDescriptor | None = None,
+) -> list[OutputArtifactDeclaration]:
+    if not isinstance(value, dict) or set(value) - {
+        "pages",
+        "tables",
+        "artifacts",
+        "metadata",
+    }:
+        raise invalid_output("Document 输出结构无效")
+    pages = value.get("pages")
+    if not isinstance(pages, list):
+        raise invalid_output("Document 输出缺少页级结果")
+    if len(pages) > max_units:
+        raise invalid_output("Document 输出页数超过上限")
+    page_numbers: set[int] = set()
+    for page in pages:
+        if not isinstance(page, dict) or set(page) - {
+            "pageNumber",
+            "text",
+            "blocks",
+            "layout",
+        }:
+            raise invalid_output("Document 页级结果无效")
+        page_number = page.get("pageNumber")
+        if (
+            not integer(page_number)
+            or page_number < 1
+            or page_number > max_units
+            or page_number in page_numbers
+            or not isinstance(page.get("text"), str)
+            or not isinstance(page.get("blocks"), list)
+        ):
+            raise invalid_output("Document 页码、文本或 Block 无效")
+        page_numbers.add(page_number)
+        blocks = page["blocks"]
+        if not isinstance(blocks, list):
+            raise invalid_output("Document Block 无效")
+        for block in blocks:
+            validate_block(block)
+        layout = page.get("layout")
+        if layout is not None:
+            if not isinstance(layout, list):
+                raise invalid_output("Document 版面树无效")
+            for node in layout:
+                validate_layout(node, 0)
+    validate_tables(value.get("tables", []), page_numbers)
+    artifacts = validate_artifacts(value.get("artifacts", []), sink)
+    metadata = value.get("metadata", {})
+    if not isinstance(metadata, dict) or any(
+        not scalar(item) for item in metadata.values()
+    ):
+        raise invalid_output("Document 元数据无效")
+    return artifacts
+
+
+def replace_artifacts(
+    value: JsonValue,
+    references: list[OutputArtifactReference],
+) -> JsonValue:
+    if not isinstance(value, dict):
+        raise invalid_output("Document 输出结构无效")
+    return {
+        **value,
+        "artifacts": [reference.model_dump(by_alias=True) for reference in references],
+    }
+
+
+def validate_artifacts(
+    value: object,
+    sink: OutputArtifactSinkDescriptor | None,
+) -> list[OutputArtifactDeclaration]:
+    if not isinstance(value, list):
+        raise invalid_output("Document Artifact manifest 无效")
+    if not value:
+        return []
+    if sink is None:
+        raise invalid_output("Document Adapter 不得直接签发 Artifact")
+    try:
+        artifacts = [OutputArtifactDeclaration.model_validate(item) for item in value]
+    except ValidationError:
+        raise invalid_output("Document Artifact manifest 无效") from None
+    if (
+        len(artifacts) > sink.limits.max_artifacts
+        or len({item.output_key for item in artifacts}) != len(artifacts)
+        or sum(item.byte_size for item in artifacts) > sink.limits.max_total_bytes
+        or any(
+            item.byte_size > sink.limits.max_artifact_bytes
+            or item.media_type not in sink.allowed_media_types
+            for item in artifacts
+        )
+    ):
+        raise invalid_output("Document Artifact 超过 Sink 上限")
+    return artifacts
+
+
+def validate_tables(value: object, page_numbers: set[int]) -> None:
+    if not isinstance(value, list):
+        raise invalid_output("Document 表格结果无效")
+    for table in value:
+        rows = table.get("rows") if isinstance(table, dict) else None
+        if (
+            not isinstance(table, dict)
+            or set(table) - {"pageNumber", "rows", "bounds"}
+            or table.get("pageNumber") not in page_numbers
+            or not isinstance(rows, list)
+            or any(
+                not isinstance(row, list)
+                or any(not isinstance(cell, str) for cell in row)
+                for row in rows
+            )
+        ):
+            raise invalid_output("Document 表格结果无效")
+        validate_bounds(table.get("bounds"))
+
+
+def validate_block(value: object) -> None:
+    if (
+        not isinstance(value, dict)
+        or set(value) - {"id", "text", "confidence", "bounds"}
+        or not isinstance(value.get("id"), str)
+        or not isinstance(value.get("text"), str)
+    ):
+        raise invalid_output("Document 文本 Block 无效")
+    confidence = value.get("confidence")
+    if confidence is not None and (
+        not number(confidence) or confidence < 0 or confidence > 1
+    ):
+        raise invalid_output("Document Block 置信度无效")
+    validate_bounds(value.get("bounds"))
+
+
+def validate_layout(value: object, depth: int) -> None:
+    if (
+        depth > 32
+        or not isinstance(value, dict)
+        or set(value) - {"id", "type", "text", "bounds", "children"}
+    ):
+        raise invalid_output("Document 版面树无效")
+    if not isinstance(value.get("id"), str) or not isinstance(value.get("type"), str):
+        raise invalid_output("Document 版面节点无效")
+    text = value.get("text")
+    if text is not None and not isinstance(text, str):
+        raise invalid_output("Document 版面节点文本无效")
+    validate_bounds(value.get("bounds"))
+    children = value.get("children", [])
+    if not isinstance(children, list):
+        raise invalid_output("Document 版面子节点无效")
+    for child in children:
+        validate_layout(child, depth + 1)
+
+
+def validate_bounds(value: object) -> None:
+    if value is None:
+        return
+    if (
+        not isinstance(value, dict)
+        or set(value) != {"x", "y", "width", "height", "unit"}
+        or value.get("unit") not in {"pixel", "normalized"}
+    ):
+        raise invalid_output("Document 坐标无效")
+    for field in ("x", "y", "width", "height"):
+        coordinate = value.get(field)
+        if not number(coordinate) or coordinate < 0:
+            raise invalid_output("Document 坐标无效")
+
+
+def integer(value: object) -> TypeGuard[int]:
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def number(value: object) -> TypeGuard[int | float]:
+    return (
+        isinstance(value, int | float)
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+    )
+
+
+def scalar(value: object) -> bool:
+    return isinstance(value, str | bool) or number(value)
+
+
+__all__ = ["SafeDocumentError", "replace_artifacts", "validate_output"]

package/template/apps/document-server/src/southwind_document_server/progress.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from .artifacts import AdapterExecution
+from .store import DocumentRunStore
+
+
+def record_page_progress(
+    store: DocumentRunStore,
+    run_id: str,
+    execution: AdapterExecution,
+) -> None:
+    output = execution.output
+    pages = output.get("pages") if isinstance(output, dict) else None
+    if not isinstance(pages, list):
+        return
+    total = len(pages)
+    for index in range(total):
+        if not store.append_progress(
+            run_id,
+            "document.page.completed",
+            {
+                "pageNumber": index + 1,
+                "completedPages": index + 1,
+                "totalPages": total,
+            },
+        ):
+            return