@scrypted/libav 1.0.167 → 1.0.169

package/opus/training/rnn_train.py

@@ -0,0 +1,177 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+from keras.models import Sequential
+from keras.models import Model
+from keras.layers import Input
+from keras.layers import Dense
+from keras.layers import LSTM
+from keras.layers import GRU
+from keras.layers import CuDNNGRU
+from keras.layers import SimpleRNN
+from keras.layers import Dropout
+from keras import losses
+import h5py
+from keras.optimizers import Adam
+
+from keras.constraints import Constraint
+from keras import backend as K
+import numpy as np
+
+import tensorflow as tf
+from keras.backend.tensorflow_backend import set_session
+config = tf.ConfigProto()
+config.gpu_options.per_process_gpu_memory_fraction = 0.44
+set_session(tf.Session(config=config))
+
+def binary_crossentrop2(y_true, y_pred):
+    return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_true, y_pred), axis=-1)
+
+def binary_accuracy2(y_true, y_pred):
+    return K.mean(K.cast(K.equal(y_true, K.round(y_pred)), 'float32') + K.cast(K.equal(y_true, 0.5), 'float32'), axis=-1)
+
+def quant_model(model):
+    weights = model.get_weights()
+    for k in range(len(weights)):
+        weights[k] = np.maximum(-128, np.minimum(127, np.round(128*weights[k])*0.0078125))
+    model.set_weights(weights)
+
+class WeightClip(Constraint):
+    '''Clips the weights incident to each hidden unit to be inside a range
+    '''
+    def __init__(self, c=2):
+        self.c = c
+
+    def __call__(self, p):
+        return K.clip(p, -self.c, self.c)
+
+    def get_config(self):
+        return {'name': self.__class__.__name__,
+            'c': self.c}
+
+reg = 0.000001
+constraint = WeightClip(.998)
+
+print('Build model...')
+
+main_input = Input(shape=(None, 25), name='main_input')
+x = Dense(32, activation='tanh', kernel_constraint=constraint, bias_constraint=constraint)(main_input)
+#x = CuDNNGRU(24, return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x)
+x = GRU(24, recurrent_activation='sigmoid', activation='tanh', return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x)
+x = Dense(2, activation='sigmoid', kernel_constraint=constraint, bias_constraint=constraint)(x)
+model = Model(inputs=main_input, outputs=x)
+
+batch_size = 2048
+
+print('Loading data...')
+with h5py.File('features10b.h5', 'r') as hf:
+    all_data = hf['data'][:]
+print('done.')
+
+window_size = 1500
+
+nb_sequences = len(all_data)//window_size
+print(nb_sequences, ' sequences')
+x_train = all_data[:nb_sequences*window_size, :-2]
+x_train = np.reshape(x_train, (nb_sequences, window_size, 25))
+
+y_train = np.copy(all_data[:nb_sequences*window_size, -2:])
+y_train = np.reshape(y_train, (nb_sequences, window_size, 2))
+
+print("Marking ignores")
+for s in y_train:
+    for e in s:
+        if (e[1] >= 1):
+            break
+        e[0] = 0.5
+
+all_data = 0;
+x_train = x_train.astype('float32')
+y_train = y_train.astype('float32')
+
+print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape)
+
+model.load_weights('newweights10a1b_ep206.hdf5')
+
+#weights = model.get_weights()
+#for k in range(len(weights)):
+#    weights[k] = np.round(128*weights[k])*0.0078125
+#model.set_weights(weights)
+
+# try using different optimizers and different optimizer configs
+model.compile(loss=binary_crossentrop2,
+              optimizer=Adam(0.0001),
+              metrics=[binary_accuracy2])
+
+print('Train...')
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=10, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep10.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=50, initial_epoch=10)
+model.save("newweights10a1c_ep50.hdf5")
+
+model.compile(loss=binary_crossentrop2,
+              optimizer=Adam(0.0001),
+              metrics=[binary_accuracy2])
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=100, initial_epoch=50)
+model.save("newweights10a1c_ep100.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=150, initial_epoch=100)
+model.save("newweights10a1c_ep150.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=200, initial_epoch=150)
+model.save("newweights10a1c_ep200.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=201, initial_epoch=200)
+model.save("newweights10a1c_ep201.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=202, initial_epoch=201, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep202.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=203, initial_epoch=202, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep203.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=204, initial_epoch=203, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep204.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=205, initial_epoch=204, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep205.hdf5")
+
+quant_model(model)
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=206, initial_epoch=205, validation_data=(x_train, y_train))
+model.save("newweights10a1c_ep206.hdf5")
+

package/opus/training/txt2hdf5.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+import numpy as np
+import h5py
+import sys
+
+data = np.loadtxt(sys.argv[1], dtype='float32')
+h5f = h5py.File(sys.argv[2], 'w');
+h5f.create_dataset('data', data=data)
+h5f.close()

package/opus/update_version

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Creates and updates the package_version information used by configure.ac
+# (or other makefiles).  When run inside a git repository it will use the
+# version information that can be queried from it unless AUTO_UPDATE is set
+# to 'no'.  If no version is currently known it will be set to 'unknown'.
+#
+# If called with the argument 'release', the PACKAGE_VERSION will be updated
+# even if AUTO_UPDATE=no, but the value of AUTO_UPDATE shall be preserved.
+# This is used to force a version update whenever `make dist` is run.
+#
+# The exit status is 1 if package_version is not modified, else 0 is returned.
+#
+# This script should NOT be included in distributed tarballs, because if a
+# parent directory contains a git repository we do not want to accidentally
+# retrieve the version information from it instead.  Tarballs should ship
+# with only the package_version file.
+#
+# Ron <ron@debian.org>, 2012.
+
+SRCDIR=$(dirname $0)
+
+if [ -e "$SRCDIR/package_version" ]; then
+    . "$SRCDIR/package_version"
+fi
+
+if [ "$AUTO_UPDATE" = no ]; then
+    [ "$1" = release ] || exit 1
+else
+    AUTO_UPDATE=yes
+fi
+
+# We run `git status` before describe here to ensure that we don't get a false
+# -dirty from files that have been touched but are not actually altered in the
+# working dir.
+GIT_VERSION=$(cd "$SRCDIR" && git status > /dev/null 2>&1 \
+                           && git describe --tags --match 'v*' --dirty 2> /dev/null)
+GIT_VERSION=${GIT_VERSION#v}
+
+if [ -n "$GIT_VERSION" ]; then
+
+    [ "$GIT_VERSION" != "$PACKAGE_VERSION" ] || exit 1
+    PACKAGE_VERSION="$GIT_VERSION"
+
+elif [ -z "$PACKAGE_VERSION" ]; then
+    # No current package_version and no git ...
+    # We really shouldn't ever get here, because this script should only be
+    # included in the git repository, and should usually be export-ignored.
+    PACKAGE_VERSION="unknown"
+else
+    exit 1
+fi
+
+cat > "$SRCDIR/package_version" <<-EOF
+	# Automatically generated by update_version.
+	# This file may be sourced into a shell script or makefile.
+
+	# Set this to 'no' if you do not wish the version information
+	# to be checked and updated for every build.  Most people will
+	# never want to change this, it is an option for developers
+	# making frequent changes that they know will not be released.
+	AUTO_UPDATE=$AUTO_UPDATE
+
+	PACKAGE_VERSION="$PACKAGE_VERSION"
+EOF

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrypted/libav",
-  "version": "1.0.167",
+  "version": "1.0.169",
   "dependencies": {
     "detect-libc": "^2.0.3",
     "follow-redirects": "^1.15.9",
@@ -16,7 +16,7 @@
   "scripts": {
     "prebuildify-host": "prebuild --strip --include-regex \"\\.node$|ffmpeg$|\\.exe$|\\.dll$|\\.pdb$\"",
     "prebuildify-node": "prebuild --strip -t 20.0.0 -t 22.0.0 --include-regex \"\\.node$|ffmpeg$|\\.exe$|\\.dll$|\\.pdb$\"",
-    "prebuildify-electron": "prebuild --strip -r electron -t 31.0.0 -t 33.0.0 -t 35.0.0 --include-regex \"\\.node$|ffmpeg$|\\.exe$|\\.dll$|\\.pdb$\"",
+    "prebuildify-electron": "prebuild --strip -r electron -t 31.0.0 -t 33.0.0 --include-regex \"\\.node$|ffmpeg$|\\.exe$|\\.dll$|\\.pdb$\"",
     "prebuildify-arm64": "npm run prebuildify-electron -- --arch arm64 && npm run prebuildify-node -- --arch arm64",
     "prebuildify-x64": "npm run prebuildify-electron -- --arch x64 && npm run prebuildify-node -- --arch x64",
     "prebuildify": "npm run prebuildify-arm64 && npm run prebuildify-x64",

package/src/build-ffmpeg-msys.sh

@@ -9,6 +9,12 @@ cmake -B _build -DCMAKE_INSTALL_PREFIX=$VPL_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
 cmake --build _build --config Release
 cmake --install _build --config Release
 
+cd ../opus
+export OPUS_INSTALL_DIR=$PWD/../_opusinstall
+cmake -B _build -DCMAKE_INSTALL_PREFIX=$OPUS_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DUSE_MSVC_STATIC_RUNTIME=ON -DMINGW_LIBS="-ladvapi32 -lmsvcrt -lole32"
+cmake --build _build --config Release
+cmake --install _build --config Release
+
 cd ../FFmpeg
 
 if [ ! -z "$FFMPEG_NO_REBUILD" ]
@@ -22,7 +28,7 @@ fi
 
 # yasm can be dropped
 pacman --noconfirm -S yasm nasm pkg-config clang diffutils make
-export PKG_CONFIG_PATH=$PWD/../_vplinstall/lib/pkgconfig:$PWD/../_nvinstall/lib/pkgconfig:$PWD/../src/pkgconfig:$PKG_CONFIG_PATH
+export PKG_CONFIG_PATH=$PWD/../_opusinstall/lib/pkgconfig:$PWD/../_vplinstall/lib/pkgconfig:$PWD/../_nvinstall/lib/pkgconfig:$PWD/../src/pkgconfig
 
 # the --enable-nvdec is necessary for build sanity checking, the build will configure
 # successfully if nvidia stuff is missing, even though it is required for --enable-cuda-llvm
@@ -31,5 +37,5 @@ export PKG_CONFIG_PATH=$PWD/../_vplinstall/lib/pkgconfig:$PWD/../_nvinstall/lib/
 # this version is the last that has HLSL and OGLCompiler which are still required by ffmpeg for now
 # https://sdk.lunarg.com/sdk/download/1.3.268.0/windows/VulkanSDK-1.3.268.0-Installer.exe
 
-./configure --enable-libvpl --enable-libglslang --enable-cuda-llvm --enable-nvdec --toolchain=msvc && make -j32
+./configure --enable-libopus --enable-libvpl --enable-libglslang --enable-cuda-llvm --enable-nvdec --toolchain=msvc && make -j32
 exit $?