rays-video 0.1.2

data/src/osx/video_audio_in.mm

@@ -0,0 +1,252 @@
+// -*- mode: objc -*-
+#import "video_audio_in.h"
+
+
+#import <AVFoundation/AVFoundation.h>
+#include "rays/exception.h"
+
+
+namespace Rays
+{
+
+
+	struct VideoAudioIn::Data
+	{
+
+		AVAsset* asset            = nil;
+
+		AVAssetTrack* audio_track = nil;
+
+		uint nsamples = 0;
+
+		Beeps::Signals buffer;
+
+		uint           buffer_offset = 0;
+
+		AVAssetReader* reader               = nil;
+
+		AVAssetReaderAudioMixOutput* output = nil;
+
+		Data (AVAsset* asset, AVAssetTrack* audio_track)
+		{
+			assert(asset && audio_track);
+
+			CMFormatDescriptionRef format =
+				(__bridge CMFormatDescriptionRef) audio_track.formatDescriptions.firstObject;
+			if (!format)
+				rays_error(__FILE__, __LINE__, "failed to get CMFormatDescription");
+
+			const AudioStreamBasicDescription* desc =
+				CMAudioFormatDescriptionGetStreamBasicDescription(format);
+			if (!desc)
+				rays_error(__FILE__, __LINE__, "failed to get AudioStreamBasicDescription");
+
+			this->asset       = [asset       retain];
+			this->audio_track = [audio_track retain];
+			uint nchannels    = std::min<uint>(desc->mChannelsPerFrame, 2);
+			this->buffer      = Beeps::Signals(2048, nchannels, desc->mSampleRate);
+			double duration   = CMTimeGetSeconds(audio_track.timeRange.duration);
+			this->nsamples    = (uint) (duration * buffer.sample_rate());
+		}
+
+		~Data ()
+		{
+			clear_reader();
+			[audio_track release];
+			[asset       release];
+		}
+
+		void create_reader (uint offset)
+		{
+			clear_reader();
+
+			NSError* error        = nil;
+			AVAssetReader* reader =
+				[[[AVAssetReader alloc] initWithAsset: asset error: &error] autorelease];
+			if (!reader || error)
+				rays_error(__FILE__, __LINE__, "failed to create AVAssetReader");
+
+			AVAssetReaderAudioMixOutput* output = [AVAssetReaderAudioMixOutput
+				assetReaderAudioMixOutputWithAudioTracks: @[audio_track]
+				audioSettings: @{
+					AVFormatIDKey:               @(kAudioFormatLinearPCM),
+					AVLinearPCMBitDepthKey:      @32,
+					AVLinearPCMIsFloatKey:       @YES,
+					AVLinearPCMIsNonInterleaved: @YES,
+					AVNumberOfChannelsKey:       @(buffer.nchannels()),
+				}];
+			if (![reader canAddOutput: output])
+				rays_error(__FILE__, __LINE__, "cannot add audio output");
+
+			[reader addOutput: output];
+			reader.timeRange = CMTimeRangeMake(
+				CMTimeMakeWithSeconds(
+					(double) offset / buffer.sample_rate(),
+					(int32_t) buffer.sample_rate()),
+				kCMTimePositiveInfinity);
+			if (![reader startReading])
+				rays_error(__FILE__, __LINE__, "failed to start reading audio");
+
+			this->reader        = [reader retain];
+			this->output        = [output retain];
+			this->buffer_offset = offset;
+			this->buffer        .clear();
+		}
+
+		void clear_reader ()
+		{
+			if (reader && reader.status == AVAssetReaderStatusReading)
+				[reader cancelReading];
+
+			[output release];
+			output = nil;
+			[reader release];
+			reader = nil;
+		}
+
+		bool read_next (Beeps::Signals* signals, uint* offset)
+		{
+			assert(signals && offset);
+
+			if (
+				!reader ||
+				*offset < buffer_offset ||
+				*offset > buffer_offset + buffer.nsamples())
+			{
+				create_reader(*offset);
+			}
+
+			if (*offset == buffer_offset + buffer.nsamples())
+			{
+				if (!read_next_buffer()) return false;
+				buffer_offset = *offset;
+			}
+
+			uint size = signals->append(buffer, *offset - buffer_offset);
+			*offset += size;
+			return size > 0;
+		}
+
+		bool read_next_buffer ()
+		{
+			if (!reader || !output || reader.status != AVAssetReaderStatusReading)
+				return false;
+
+			std::shared_ptr<opaqueCMSampleBuffer> samples(
+				[output copyNextSampleBuffer],
+				CFRelease);
+			if (!samples)
+				return false;
+
+			uint nsamples = (uint) CMSampleBufferGetNumSamples(samples.get());
+			if (nsamples <= 0)
+				return false;
+
+			CMBlockBufferRef block = CMSampleBufferGetDataBuffer(samples.get());
+			if (!block)
+				return false;
+
+			size_t size = CMBlockBufferGetDataLength(block);
+			char* data  = NULL;
+			CMBlockBufferGetDataPointer(block, 0, NULL, &size, &data);
+			if (!data || size <= 0)
+				rays_error(__FILE__, __LINE__);
+
+			CMFormatDescriptionRef format =
+				CMSampleBufferGetFormatDescription(samples.get());
+			if (!format)
+				rays_error(__FILE__, __LINE__);
+
+			const AudioStreamBasicDescription* desc =
+				CMAudioFormatDescriptionGetStreamBasicDescription(format);
+			if (!desc)
+				rays_error(__FILE__, __LINE__);
+
+			uint nchannels = desc->mChannelsPerFrame;
+			if (nchannels != buffer.nchannels())
+				rays_error(__FILE__, __LINE__);
+
+			std::vector<const float*> channels(nchannels);
+			for (uint ch = 0; ch < nchannels; ++ch)
+				channels[ch] = (const float*) data + ch * nsamples;
+
+			buffer.clear(nsamples);
+			buffer.append(channels.data(), nsamples, nchannels, buffer.sample_rate());
+			return buffer.nsamples() > 0;
+		}
+
+	};// VideoAudioIn::Data
+
+
+	VideoAudioIn::Data*
+	VideoAudioIn_Data_create (AVAsset* asset, AVAssetTrack* audio_track)
+	{
+		return new VideoAudioIn::Data(asset, audio_track);
+	}
+
+
+	VideoAudioIn::VideoAudioIn (Data* data)
+	:	self(data)
+	{
+	}
+
+	VideoAudioIn::~VideoAudioIn ()
+	{
+	}
+
+	double
+	VideoAudioIn::sample_rate () const
+	{
+		return self->buffer.sample_rate();
+	}
+
+	uint
+	VideoAudioIn::nchannels () const
+	{
+		return self->buffer.nchannels();
+	}
+
+	uint
+	VideoAudioIn::nsamples () const
+	{
+		return self->nsamples;
+	}
+
+	float
+	VideoAudioIn::seconds () const
+	{
+		if (this->sample_rate() <= 0)
+			return 0;
+
+		return (float) (self->nsamples / this->sample_rate());
+	}
+
+	VideoAudioIn::operator bool () const
+	{
+		return
+			Super::operator bool() &&
+			self->asset &&
+			self->audio_track &&
+			self->buffer;
+	}
+
+	bool
+	VideoAudioIn::seekable () const
+	{
+		return true;
+	}
+
+	void
+	VideoAudioIn::generate (Context* context, Beeps::Signals* signals, uint* offset)
+	{
+		Super::generate(context, signals, offset);
+
+		while (!signals->full())
+		{
+			if (!self->read_next(signals, offset))
+				break;
+		}
+	}
+
+
+}// Rays

data/src/sdl/video.cpp

@@ -0,0 +1,86 @@
+#include "../video.h"
+
+
+#include "rays/exception.h"
+
+
+namespace Rays
+{
+
+
+	struct VideoReader::Data
+	{
+	};// VideoReader::Data
+
+
+	VideoReader::VideoReader ()
+	{
+	}
+
+	VideoReader::VideoReader (const char*)
+	{
+		not_implemented_error(__FILE__, __LINE__);
+	}
+
+	Image
+	VideoReader::decode_image (size_t, float) const
+	{
+		not_implemented_error(__FILE__, __LINE__);
+	}
+
+	VideoAudioInList
+	VideoReader::get_audio_tracks () const
+	{
+		return {};
+	}
+
+	coord
+	VideoReader::width () const
+	{
+		return 0;
+	}
+
+	coord
+	VideoReader::height () const
+	{
+		return 0;
+	}
+
+	float
+	VideoReader::fps () const
+	{
+		return 0;
+	}
+
+	size_t
+	VideoReader::size () const
+	{
+		return 0;
+	}
+
+	VideoReader::operator bool () const
+	{
+		return false;
+	}
+
+	bool
+	VideoReader::operator ! () const
+	{
+		return !operator bool();
+	}
+
+
+	void
+	Video::save (const char*)
+	{
+		not_implemented_error(__FILE__, __LINE__);
+	}
+
+	const StringList&
+	get_video_exts ()
+	{
+		not_implemented_error(__FILE__, __LINE__);
+	}
+
+
+}// Rays

data/src/sdl/video_audio_in.cpp

@@ -0,0 +1,63 @@
+#include "../video_audio_in.h"
+
+
+namespace Rays
+{
+
+
+	struct VideoAudioIn::Data
+	{
+	};// VideoAudioIn::Data
+
+
+	VideoAudioIn::VideoAudioIn (Data* data)
+	:	self(data)
+	{
+	}
+
+	VideoAudioIn::~VideoAudioIn ()
+	{
+	}
+
+	double
+	VideoAudioIn::sample_rate () const
+	{
+		return 0;
+	}
+
+	uint
+	VideoAudioIn::nchannels () const
+	{
+		return 0;
+	}
+
+	uint
+	VideoAudioIn::nsamples () const
+	{
+		return 0;
+	}
+
+	float
+	VideoAudioIn::seconds () const
+	{
+		return 0;
+	}
+
+	VideoAudioIn::operator bool () const
+	{
+		return false;
+	}
+
+	bool
+	VideoAudioIn::seekable () const
+	{
+		return false;
+	}
+
+	void
+	VideoAudioIn::generate (Context* context, Beeps::Signals* signals, uint* offset)
+	{
+	}
+
+
+}// Rays

data/src/video.cpp

@@ -0,0 +1,278 @@
+#include "video.h"
+
+
+#include <xot/util.h>
+#include <beeps/sound.h>
+#include "rays/bitmap.h"
+#include "rays/exception.h"
+
+
+namespace Rays
+{
+
+
+	struct Video::Data
+	{
+
+		int width = 0, height = 0;
+
+		float fps = 0, pixel_density = 1;
+
+		size_t position = 0;
+
+		std::vector<Image> images;
+
+		VideoAudioInList audio_tracks;
+
+		Beeps::SoundPlayer player;
+
+	};// Video::Data
+
+
+	struct VideoImageData : public Image::Data
+	{
+
+		VideoReader reader;
+
+		size_t index;
+
+		VideoImageData (const VideoReader& reader, size_t index)
+		:	reader(reader), index(index)
+		{
+		}
+
+		void preprocess (const Image* image) const override
+		{
+			Image decoded = reader.decode_image(index, 1);
+			Bitmap bitmap = decoded.bitmap();
+			if (bitmap) Xot::hint_memory_usage(bitmap.size());
+			const_cast<Image*>(image)->self = decoded.self;
+		}
+
+	};// VideoImageData
+
+
+	Video
+	load_video (const char* path)
+	{
+		VideoReader reader(path);
+		if (!reader)
+			invalid_state_error(__FILE__, __LINE__);
+
+		Video video;
+		Video::Data* self   = video.self.get();
+		self->width         = reader.width();
+		self->height        = reader.height();
+		self->fps           = reader.fps();
+		self->pixel_density = 1;
+		self->position      = 0;
+
+		size_t size = reader.size();
+		self->images.reserve(size);
+		for (size_t i = 0; i < size; ++i)
+			self->images.push_back(Image(new VideoImageData(reader, i)));
+
+		self->audio_tracks = reader.get_audio_tracks();
+
+		return video;
+	}
+
+
+	Video::Video ()
+	{
+	}
+
+	Video::Video (int width, int height, float fps, float pixel_density)
+	{
+		if (width         <= 0)
+			argument_error(__FILE__, __LINE__,         "width must be > 0");
+		if (height        <= 0)
+			argument_error(__FILE__, __LINE__,        "height must be > 0");
+		if (fps           <= 0)
+			argument_error(__FILE__, __LINE__,           "fps must be > 0");
+		if (pixel_density <= 0)
+			argument_error(__FILE__, __LINE__, "pixel_density must be > 0");
+
+		self->width         = width;
+		self->height        = height;
+		self->fps           = fps;
+		self->pixel_density = pixel_density;
+	}
+
+	Video::~Video ()
+	{
+	}
+
+	Video
+	Video::dup () const
+	{
+		Video v;
+		*v.self = *self;
+		return v;
+	}
+
+	void
+	Video::insert (size_t index, const Image& image)
+	{
+		if (!*this)
+			invalid_state_error(__FILE__, __LINE__, "video is not initialized");
+
+		self->images.insert(self->images.begin() + index, image);
+	}
+
+	void
+	Video::append (const Image& image)
+	{
+		insert(size(), image);
+	}
+
+	void
+	Video::remove (size_t index)
+	{
+		if (index >= size()) return;
+		self->images.erase(self->images.begin() + index);
+	}
+
+	void
+	Video::play ()
+	{
+		if (empty())
+			invalid_state_error(__FILE__, __LINE__, "video is empty");
+
+		if (self->audio_tracks.empty())
+			invalid_state_error(__FILE__, __LINE__, "playing video without audio is not yet supported");
+
+		VideoAudioIn* in = self->audio_tracks[0].get();
+		self->player = Beeps::Sound(in, 0, in->nchannels(), in->sample_rate()).play();
+	}
+
+	void
+	Video::pause ()
+	{
+		if (self->player) self->player.pause();
+	}
+
+	void
+	Video::stop ()
+	{
+		if (self->player) self->player.stop();
+	}
+
+	void
+	Video::set_time_scale (float scale)
+	{
+		if (self->player) self->player.set_time_scale(scale);
+	}
+
+	float
+	Video::time_scale () const
+	{
+		return self->player ? self->player.time_scale() : 1;
+	}
+
+	coord
+	Video::width () const
+	{
+		return self->width;
+	}
+
+	coord
+	Video::height () const
+	{
+		return self->height;
+	}
+
+	float
+	Video::fps () const
+	{
+		return self->fps;
+	}
+
+	float
+	Video::pixel_density () const
+	{
+		return self->pixel_density;
+	}
+
+	size_t
+	Video::size () const
+	{
+		return self->images.size();
+	}
+
+	bool
+	Video::empty () const
+	{
+		return self->images.empty();
+	}
+
+	void
+	Video::set_position (size_t index)
+	{
+		     if (empty())         index = 0;
+		else if (index >= size()) index = size() - 1;
+		self->position = index;
+	}
+
+	size_t
+	Video::position () const
+	{
+		     if (empty())                  self->position = 0;
+		else if (self->position >= size()) self->position = size() - 1;
+		return self->position;
+	}
+
+	Video::const_iterator
+	Video::begin () const
+	{
+		return self->images.begin();
+	}
+
+	Video::const_iterator
+	Video::end () const
+	{
+		return self->images.end();
+	}
+
+	Image
+	Video::operator [] (size_t index) const
+	{
+		if (empty())
+		{
+			index_error(
+				__FILE__, __LINE__, "index %zu is out of range (empty)",
+				index);
+		}
+		if (index >= size())
+		{
+			index_error(
+				__FILE__, __LINE__, "index %zu is out of range (0..%zu)",
+				index, size() - 1);
+		}
+		return self->images[index];
+	}
+
+	Video::operator Image () const
+	{
+		if (self->player)
+		{
+			size_t index = (size_t) (self->player.time() * self->fps);
+			if (index >= size()) index = size() - 1;
+			self->position = index;
+		}
+		return operator[](self->position);
+	}
+
+	Video::operator bool () const
+	{
+		return self->width > 0 && self->height > 0;
+	}
+
+	bool
+	Video::operator ! () const
+	{
+		return !operator bool();
+	}
+
+
+}// Rays

data/src/video.h

@@ -0,0 +1,50 @@
+// -*- c++ -*-
+#pragma once
+#ifndef __RAYS_VIDEO_SRC_VIDEO_H__
+#define __RAYS_VIDEO_SRC_VIDEO_H__
+
+
+#include "rays/video.h"
+#include "video_audio_in.h"
+
+
+namespace Rays
+{
+
+
+	class VideoReader
+	{
+
+		public:
+
+			VideoReader ();
+
+			VideoReader (const char* path);
+
+			Image decode_image (size_t index, float pixel_density) const;
+
+			VideoAudioInList get_audio_tracks () const;
+
+			coord width () const;
+
+			coord height () const;
+
+			float fps () const;
+
+			size_t size () const;
+
+			operator bool () const;
+
+			bool operator ! () const;
+
+			struct Data;
+
+			Xot::PSharedImpl<Data> self;
+
+	};// VideoReader
+
+
+}// Rays
+
+
+#endif//EOH